diff --git a/.ci/README b/.ci/README deleted file mode 100644 index 86b72afb83..0000000000 --- a/.ci/README +++ /dev/null @@ -1,3 +0,0 @@ -This directory contains support scripts for Travis and Appveyor continuous -integration services. -Travis is used to run tests on Linux and OSX, Appveyor runs tests on Windows. diff --git a/.ci/appveyor/README b/.ci/appveyor/README deleted file mode 100644 index 2e092a07c8..0000000000 --- a/.ci/appveyor/README +++ /dev/null @@ -1,2 +0,0 @@ -This directory contains support files for appveyor, a continuous integration -service which runs tests on Windows on every push. diff --git a/.ci/appveyor/download_exes.py b/.ci/appveyor/download_exes.py deleted file mode 100755 index 37ebdfd147..0000000000 --- a/.ci/appveyor/download_exes.py +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -""" -Script which downloads exe and wheel files hosted on AppVeyor: -https://ci.appveyor.com/project/giampaolo/psutil -Copied and readapted from the original recipe of Ibarra Corretge' -: -http://code.saghul.net/index.php/2015/09/09/ -""" - -from __future__ import print_function -import argparse -import errno -import multiprocessing -import os -import requests -import shutil -import sys - -from concurrent.futures import ThreadPoolExecutor - - -BASE_URL = 'https://ci.appveyor.com/api' -PY_VERSIONS = ['2.7', '3.3', '3.4', '3.5'] - - -def term_supports_colors(file=sys.stdout): - try: - import curses - assert file.isatty() - curses.setupterm() - assert curses.tigetnum("colors") > 0 - except Exception: - return False - else: - return True - - -if term_supports_colors(): - def hilite(s, ok=True, bold=False): - """Return an highlighted version of 'string'.""" - attr = [] - if ok is None: # no color - pass - elif ok: # green - attr.append('32') - else: # red - attr.append('31') - if bold: - attr.append('1') - return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), s) -else: - def hilite(s, *a, **k): - return s - - -def safe_makedirs(path): - try: - os.makedirs(path) - except OSError as err: - if err.errno == errno.EEXIST: - if not os.path.isdir(path): - raise - else: - raise - - -def safe_rmtree(path): - def onerror(fun, path, excinfo): - exc = excinfo[1] - if exc.errno != errno.ENOENT: - raise - - shutil.rmtree(path, onerror=onerror) - - -def download_file(url): - local_fname = url.split('/')[-1] - local_fname = os.path.join('dist', local_fname) - print(local_fname) - safe_makedirs('dist') - r = requests.get(url, stream=True) - with open(local_fname, 'wb') as f: - for chunk in r.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - return local_fname - - -def get_file_urls(options): - session = requests.Session() - data = session.get( - BASE_URL + '/projects/' + options.user + '/' + options.project) - data = data.json() - - urls = [] - for job in (job['jobId'] for job in data['build']['jobs']): - job_url = BASE_URL + '/buildjobs/' + job + '/artifacts' - data = session.get(job_url) - data = data.json() - for item in data: - file_url = job_url + '/' + item['fileName'] - urls.append(file_url) - if not urls: - sys.exit("no artifacts found") - for url in sorted(urls, key=lambda x: os.path.basename(x)): - yield url - - -def rename_27_wheels(): - # See: https://github.com/giampaolo/psutil/issues/810 - src = 'dist/psutil-4.3.0-cp27-cp27m-win32.whl' - dst = 'dist/psutil-4.3.0-cp27-none-win32.whl' - print("rename: %s\n %s" % (src, dst)) - os.rename(src, dst) - src = 'dist/psutil-4.3.0-cp27-cp27m-win_amd64.whl' - dst = 'dist/psutil-4.3.0-cp27-none-win_amd64.whl' - print("rename: %s\n %s" % (src, dst)) - os.rename(src, dst) - - -def main(options): - files = [] - safe_rmtree('dist') - with ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as e: - for url in get_file_urls(options): - fut = e.submit(download_file, url) - files.append(fut.result()) - # 2 exes (32 and 64 bit) and 2 wheels (32 and 64 bit) for each ver. - expected = len(PY_VERSIONS) * 4 - got = len(files) - if expected != got: - print(hilite("expected %s files, got %s" % (expected, got), ok=False), - file=sys.stderr) - rename_27_wheels() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='AppVeyor artifact downloader') - parser.add_argument('--user', required=True) - parser.add_argument('--project', required=True) - args = parser.parse_args() - main(args) diff --git a/.ci/appveyor/install.ps1 b/.ci/appveyor/install.ps1 deleted file mode 100644 index 3f05628255..0000000000 --- a/.ci/appveyor/install.ps1 +++ /dev/null @@ -1,85 +0,0 @@ -# Sample script to install Python and pip under Windows -# Authors: Olivier Grisel and Kyle Kastner -# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ - -$BASE_URL = "https://www.python.org/ftp/python/" -$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" -$GET_PIP_PATH = "C:\get-pip.py" - - -function DownloadPython ($python_version, $platform_suffix) { - $webclient = New-Object System.Net.WebClient - $filename = "python-" + $python_version + $platform_suffix + ".msi" - $url = $BASE_URL + $python_version + "/" + $filename - - $basedir = $pwd.Path + "\" - $filepath = $basedir + $filename - if (Test-Path $filename) { - Write-Host "Reusing" $filepath - return $filepath - } - - # Download and retry up to 5 times in case of network transient errors. - Write-Host "Downloading" $filename "from" $url - $retry_attempts = 3 - for($i=0; $i -lt $retry_attempts; $i++){ - try { - $webclient.DownloadFile($url, $filepath) - break - } - Catch [Exception]{ - Start-Sleep 1 - } - } - Write-Host "File saved at" $filepath - return $filepath -} - - -function InstallPython ($python_version, $architecture, $python_home) { - Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home - if (Test-Path $python_home) { - Write-Host $python_home "already exists, skipping." - return $false - } - if ($architecture -eq "32") { - $platform_suffix = "" - } else { - $platform_suffix = ".amd64" - } - $filepath = DownloadPython $python_version $platform_suffix - Write-Host "Installing" $filepath "to" $python_home - $args = "/qn /i $filepath TARGETDIR=$python_home" - Write-Host "msiexec.exe" $args - Start-Process -FilePath "msiexec.exe" -ArgumentList $args -Wait -Passthru - Write-Host "Python $python_version ($architecture) installation complete" - return $true -} - - -function InstallPip ($python_home) { - $pip_path = $python_home + "/Scripts/pip.exe" - $python_path = $python_home + "/python.exe" - if (-not(Test-Path $pip_path)) { - Write-Host "Installing pip..." - $webclient = New-Object System.Net.WebClient - $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) - Write-Host "Executing:" $python_path $GET_PIP_PATH - Start-Process -FilePath "$python_path" -ArgumentList "$GET_PIP_PATH" -Wait -Passthru - } else { - Write-Host "pip already installed." - } -} - -function InstallPackage ($python_home, $pkg) { - $pip_path = $python_home + "/Scripts/pip.exe" - & $pip_path install $pkg -} - -function main () { - InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON - InstallPip $env:PYTHON - InstallPackage $env:PYTHON wheel -} - -main diff --git a/.ci/appveyor/run_with_compiler.cmd b/.ci/appveyor/run_with_compiler.cmd deleted file mode 100644 index 5da547c499..0000000000 --- a/.ci/appveyor/run_with_compiler.cmd +++ /dev/null @@ -1,88 +0,0 @@ -:: To build extensions for 64 bit Python 3, we need to configure environment -:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) -:: -:: To build extensions for 64 bit Python 2, we need to configure environment -:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) -:: -:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific -:: environment configurations. -:: -:: Note: this script needs to be run with the /E:ON and /V:ON flags for the -:: cmd interpreter, at least for (SDK v7.0) -:: -:: More details at: -:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows -:: http://stackoverflow.com/a/13751649/163740 -:: -:: Author: Olivier Grisel -:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ -:: -:: Notes about batch files for Python people: -:: -:: Quotes in values are literally part of the values: -:: SET FOO="bar" -:: FOO is now five characters long: " b a r " -:: If you don't want quotes, don't include them on the right-hand side. -:: -:: The CALL lines at the end of this file look redundant, but if you move them -:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y -:: case, I don't know why. -@ECHO OFF - -SET COMMAND_TO_RUN=%* -SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows -SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf - -:: Extract the major and minor versions, and allow for the minor version to be -:: more than 9. This requires the version number to have two dots in it. -SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1% -IF "%PYTHON_VERSION:~3,1%" == "." ( - SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1% -) ELSE ( - SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2% -) - -:: Based on the Python version, determine what SDK version to use, and whether -:: to set the SDK for 64-bit. -IF %MAJOR_PYTHON_VERSION% == 2 ( - SET WINDOWS_SDK_VERSION="v7.0" - SET SET_SDK_64=Y -) ELSE ( - IF %MAJOR_PYTHON_VERSION% == 3 ( - SET WINDOWS_SDK_VERSION="v7.1" - IF %MINOR_PYTHON_VERSION% LEQ 4 ( - SET SET_SDK_64=Y - ) ELSE ( - SET SET_SDK_64=N - IF EXIST "%WIN_WDK%" ( - :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/ - REN "%WIN_WDK%" 0wdf - ) - ) - ) ELSE ( - ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" - EXIT 1 - ) -) - -IF %PYTHON_ARCH% == 64 ( - IF %SET_SDK_64% == Y ( - ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture - SET DISTUTILS_USE_SDK=1 - SET MSSdk=1 - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 - ) ELSE ( - ECHO Using default MSVC build environment for 64 bit architecture - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 - ) -) ELSE ( - ECHO Using default MSVC build environment for 32 bit architecture - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 -) diff --git a/.ci/travis/README b/.ci/travis/README deleted file mode 100644 index d9d5f65adf..0000000000 --- a/.ci/travis/README +++ /dev/null @@ -1,2 +0,0 @@ -This directory contains support files for Travis, a continuous integration -service which runs tests on Linux and Windows on every push. diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh deleted file mode 100755 index 5735b7a1da..0000000000 --- a/.ci/travis/install.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -set -e -set -x - -uname -a -python -c "import sys; print(sys.version)" - -if [[ "$(uname -s)" == 'Darwin' ]]; then - brew update || brew update - brew outdated pyenv || brew upgrade pyenv - brew install pyenv-virtualenv - - if which pyenv > /dev/null; then - eval "$(pyenv init -)" - fi - - case "${PYVER}" in - # py26) - # pyenv install 2.6.9 - # pyenv virtualenv 2.6.9 psutil - # ;; - py27) - pyenv install 2.7.10 - pyenv virtualenv 2.7.10 psutil - ;; - # py32) - # pyenv install 3.2.6 - # pyenv virtualenv 3.2.6 psutil - # ;; - # py33) - # pyenv install 3.3.6 - # pyenv virtualenv 3.3.6 psutil - # ;; - py34) - pyenv install 3.4.3 - pyenv virtualenv 3.4.3 psutil - ;; - esac - pyenv rehash - pyenv activate psutil -fi - -if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]] || [[ $PYVER == 'py26' ]]; then - pip install -U ipaddress unittest2 mock==1.0.1 -elif [[ $TRAVIS_PYTHON_VERSION == '2.7' ]] || [[ $PYVER == 'py27' ]]; then - pip install -U ipaddress mock -elif [[ $TRAVIS_PYTHON_VERSION == '3.2' ]] || [[ $PYVER == 'py32' ]]; then - pip install -U ipaddress mock -elif [[ $TRAVIS_PYTHON_VERSION == '3.3' ]] || [[ $PYVER == 'py33' ]]; then - pip install -U ipaddress -fi - -pip install coverage coveralls flake8 pep8 setuptools diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh deleted file mode 100755 index 4269f30ced..0000000000 --- a/.ci/travis/run.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -e -set -x - -if [[ "$(uname -s)" == 'Darwin' ]]; then - if which pyenv > /dev/null; then - eval "$(pyenv init -)" - fi - pyenv activate psutil -fi - -python setup.py build -python setup.py develop - -if [[ "$(uname -s)" != 'Darwin' ]]; then - coverage run psutil/tests/runner.py --include="psutil/*" --omit="test/*,*setup*" -else - python psutil/tests/runner.py -fi - -python psutil/tests/test_memory_leaks.py -flake8 -pep8 diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000000..3867ab07fa --- /dev/null +++ b/.clang-format @@ -0,0 +1,68 @@ +# Re-adapted from: https://gist.github.com/JPHutchins/6ef33a52cc92fc4a71996b32b11724b4 +# clang-format doc: https://clang.llvm.org/docs/ClangFormatStyleOptions.html + +BasedOnStyle: Google +AlignAfterOpenBracket: BlockIndent +AlignTrailingComments: false +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortEnumsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: MultiLine + AfterEnum: false + AfterExternBlock: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: true + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: false + SplitEmptyNamespace: false + SplitEmptyRecord: false +BitFieldColonSpacing: After +BreakBeforeBinaryOperators: NonAssignment +BreakBeforeBraces: Custom +BreakStringLiterals: true +ColumnLimit: 79 +DerivePointerAlignment: false +IndentCaseBlocks: true +IndentCaseLabels: true +IndentWidth: 4 +MaxEmptyLinesToKeep: 2 +PointerAlignment: Right +SortIncludes: false +SpaceBeforeParens: ControlStatementsExceptControlMacros +UseTab: Never + +# Force fun return type and fun definition to stay on 2 different lines: +# static int +# foo() { +# printf(); +# } +AlwaysBreakAfterReturnType: TopLevelDefinitions + +# Prevents: +# foo = +# Bar(...) +PenaltyBreakAssignment: 400 +PenaltyBreakBeforeFirstCallParameter: 0 + +# Handle macros with no `;` at EOL, so that they don't include the next line +# into them. +StatementMacros: + - Py_BEGIN_ALLOW_THREADS + - Py_END_ALLOW_THREADS diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 6b6309b9f0..0000000000 --- a/.coveragerc +++ /dev/null @@ -1,33 +0,0 @@ -[report] - -include = - *psutil* - -omit = - psutil/tests/* - setup.py - psutil/_compat.py - -exclude_lines = - pragma: no cover - if PY3: - if __name__ == .__main__.: - if sys.platform.startswith - if _WINDOWS: - import enum - if enum is not None: - if enum is None: - if has_enums: - if LITTLE_ENDIAN: - enum.IntEnum - except ImportError: - raise NotImplementedError - if WINDOWS - if OSX - if BSD - if FREEBSD - if OPENBSD - if NETBSD - if SUNOS - if LINUX - if ppid_map is None: diff --git a/.dprint.jsonc b/.dprint.jsonc new file mode 100644 index 0000000000..43481e5794 --- /dev/null +++ b/.dprint.jsonc @@ -0,0 +1,60 @@ +{ + "markdown": { + "lineWidth": 79, + "textWrap": "always", + }, + "json": { + "indentWidth": 4, + "associations": [ + "**/*.json", + "**/*.jsonc", + ], + }, + "yaml": { + "printWidth": 120, + "associations": [ + "**/*.yml", + "**/*.yaml", + "**/.clang-format", + ], + }, + // js + "typescript": { + "indentWidth": 4, + "lineWidth": 79, + "quoteStyle": "preferDouble", + "useBraces": "always", + "singleBodyPosition": "nextLine", + "nextControlFlowPosition": "nextLine", + "operatorPosition": "sameLine", + "conditionalExpression.operatorPosition": "nextLine", + "functionExpression.spaceBeforeParentheses": true, + "trailingCommas": "onlyMultiLine", + "associations": [ + "**/*.js", + ], + }, + // css + "malva": { + "indentWidth": 4, + "printWidth": 79, + "associations": [ + "**/*.css", + ], + }, + "excludes": [ + "**/*-lock.json", + "docs/_static/css/code.css", + "docs/_static/css/fonts.css", + ".github/ISSUE_TEMPLATE/bug.md", + ".github/ISSUE_TEMPLATE/enhancement.md", + ".github/PULL_REQUEST_TEMPLATE.md", + ], + "plugins": [ + "https://plugins.dprint.dev/markdown-0.22.1.wasm", + "https://plugins.dprint.dev/json-0.23.0.wasm", + "https://plugins.dprint.dev/g-plane/pretty_yaml-v0.6.0.wasm", + "https://plugins.dprint.dev/typescript-0.96.1.wasm", + "https://plugins.dprint.dev/g-plane/malva-v0.16.0.wasm", + ], +} diff --git a/.git-pre-commit b/.git-pre-commit deleted file mode 100755 index 99387729eb..0000000000 --- a/.git-pre-commit +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python - -# This gets executed on 'git commit' and rejects the commit in case the -# submitted code does not pass validation. -# Install it with "make install-git-hooks" - -import os -import subprocess -import sys - - -def main(): - out = subprocess.check_output("git diff --cached --name-only", shell=True) - files = [x for x in out.split(b'\n') if x.endswith(b'.py') and - os.path.exists(x)] - - for path in files: - with open(path) as f: - data = f.read() - - # pdb - if "pdb.set_trace" in data: - for lineno, line in enumerate(data.split('\n'), 1): - line = line.rstrip() - if "pdb.set_trace" in line: - print("%s: %s" % (lineno, line)) - sys.exit( - "commit aborted: you forgot a pdb in your python code") - - # bare except clause - if "except:" in data: - for lineno, line in enumerate(data.split('\n'), 1): - line = line.rstrip() - if "except:" in line and not line.endswith("# NOQA"): - print("%s: %s" % (lineno, line)) - sys.exit("commit aborted: bare except clause") - - # flake8 - failed = False - for path in files: - ret = subprocess.call("python -m flake8 %s" % path, shell=True) - if ret != 0: - failed = True - if failed: - sys.exit("commit aborted: python code is not flake8-compliant") - -main() diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000000..03c7c77c17 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,9 @@ +# These are supported funding model platforms + +tidelift: "pypi/psutil" +github: giampaolo +patreon: # Replace with a single Patreon username +open_collective: psutil +ko_fi: # Replace with a single Ko-fi username +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A9ZS7PKKRM3S8 diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md new file mode 100644 index 0000000000..7db9f45e83 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -0,0 +1,21 @@ +--- +name: Bug +about: Report a bug +title: "[OS] title" +labels: 'bug' +--- + +## Summary + +- OS: { type-or-version } +- Architecture: { 64bit, 32bit, ARM, PowerPC, s390 } +- Psutil version: { pip3 show psutil } +- Python version: { python3 -V } +- Type: { core, doc, performance, scripts, tests, wheels, new-api, installation } + +## Description + +{{{ + A clear explanation of the bug, including traceback message (if any). Please read the contributing guidelines before submit: + https://github.com/giampaolo/psutil/blob/master/CONTRIBUTING.md +}}} diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..39dc113f1a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Ask a question + url: https://groups.google.com/g/psutil + about: Use this to ask for support diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md new file mode 100644 index 0000000000..32b20a93fe --- /dev/null +++ b/.github/ISSUE_TEMPLATE/enhancement.md @@ -0,0 +1,18 @@ +--- +name: Enhancement +about: Propose an enhancement +labels: 'enhancement' +title: "[OS] title" +--- + +## Summary + +- OS: { type-or-version } +- Type: { core, doc, performance, scripts, tests, wheels, new-api } + +## Description + +{{{ + A clear explanation of your proposal. Please read the contributing guidelines before submit: + https://github.com/giampaolo/psutil/blob/master/CONTRIBUTING.md +}}} diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..dd506cb9fb --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,13 @@ +## Summary + +- OS: { type-or-version } +- Bug fix: { yes/no } +- Type: { core, doc, performance, scripts, tests, wheels, new-api } +- Fixes: { comma-separated list of issues fixed by this PR, if any } + +## Description + +{{{ + A clear explanation of your bugfix or enhancement. Please read the contributing guidelines before submit: + https://github.com/giampaolo/psutil/blob/master/CONTRIBUTING.md +}}} diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..debc712d8c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + # Bundle all action bumps into one PR instead of one each. + groups: + github-actions: + patterns: + - "*" diff --git a/.github/no-response.yml b/.github/no-response.yml new file mode 100644 index 0000000000..56457a28d0 --- /dev/null +++ b/.github/no-response.yml @@ -0,0 +1,10 @@ +# Configuration for probot-no-response: https://github.com/probot/no-response + +# Number of days of inactivity before an issue is closed for lack of response +daysUntilClose: 14 +# Label requiring a response +responseRequiredLabel: need-more-info +# Comment to post when closing an Issue for lack of response. +# Set to `false` to disable +closeComment: > + This issue has been automatically closed because there has been no response for more information from the original author. Please reach out if you have or find the answers requested so that this can be investigated further. diff --git a/.github/placeholder b/.github/placeholder new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/workflows/bsd.yml b/.github/workflows/bsd.yml new file mode 100644 index 0000000000..036cd1d5f9 --- /dev/null +++ b/.github/workflows/bsd.yml @@ -0,0 +1,80 @@ +# Execute tests on *BSD platforms. Does not produce wheels. +# Useful URLs: +# https://github.com/vmactions/freebsd-vm +# https://github.com/vmactions/openbsd-vm +# https://github.com/vmactions/netbsd-vm + +on: + workflow_dispatch: + push: + # only run this job if the following files are modified + paths: &bsd_paths + - ".github/workflows/bsd.yml" + - "Makefile" + - "psutil/__init__.py" + - "psutil/_common.py" + - "psutil/_ntuples.py" + - "psutil/_psbsd.py" + - "psutil/_psposix.py" + - "psutil/_psutil_bsd.c" + - "psutil/arch/*bsd*/**" + - "psutil/arch/*posix*/**" + - "psutil/arch/all/**" + - "pyproject.toml" + - "scripts/internal/install-pydeps.sh" + - "scripts/internal/install-sysdeps.sh" + - "setup.py" + - "tests/**" + pull_request: + paths: *bsd_paths +name: bsd +concurrency: + group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.sha || '' }} + cancel-in-progress: true +jobs: + freebsd: + # if: false + # Skip same-repo PR runs: the push event already covers them. + if: &skip_same_repo_pr github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + - name: Run tests + uses: vmactions/freebsd-vm@v1 + with: + release: "14.3" + usesh: true + run: | + make ci-test + make test-memleaks-parallel + + openbsd: + # if: false + if: *skip_same_repo_pr + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + - name: Run tests + uses: vmactions/openbsd-vm@v1 + with: + usesh: true + run: | + make ci-test + make test-memleaks-parallel + + netbsd: + # if: false + if: *skip_same_repo_pr + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + - name: Run tests + uses: vmactions/netbsd-vm@v1 + with: + usesh: true + run: | + make ci-test + make test-memleaks-parallel diff --git a/.github/workflows/changelog_bot.py b/.github/workflows/changelog_bot.py new file mode 100644 index 0000000000..dae6bff73d --- /dev/null +++ b/.github/workflows/changelog_bot.py @@ -0,0 +1,1188 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Activated when commenting "/changelog" on a PR. + +This bot will ask Claude to add an entry into docs/changelog.rst based +on the changes introduced in the PR, and also add an entry to +docs/credits.rst. + +Requires: + +- A subscription to Claude API +- The "ANTHROPIC_API_KEY" environment variable to be set via GitHub + web interface (Settings -> Secrets & Variables) +""" + +import argparse +import datetime +import json +import os +import re +import sys +import urllib.error +import urllib.request + +# CLI args +PR_NUMBER = None +REPO = None +TOKEN = None +COMMENT_FILE = None +ERROR_FILE = None + +CHANGELOG_FILE = "docs/changelog.rst" +CREDITS_FILE = "docs/credits.rst" +MODEL = "claude-sonnet-5" +MAX_DIFF_CHARS = 20_000 +COMPARE_MAX_FILES = 300 +MAX_TOKENS = 4096 # thinking tokens count against this too +HTTP_TIMEOUT = 30 + +PROMPT = """\ +You are helping maintain the official changelog for psutil, a Python +system monitoring library. + +Your task is to decide how ONE pull request should be recorded in the +changelog, and to produce at most ONE entry. + +PR #{number}: {title} +Author: @{author} (full name: {author_name}) + +Description: +{body} + +Diff: +{diff} + +CURRENT CHANGELOG BLOCK + +This is the top (in-development) version block of docs/changelog.rst. +Base your decision on what it already contains: + +----- BEGIN BLOCK ----- +{block} +----- END BLOCK ----- + +DECIDING THE ACTION + +- insert: the block has no entry for this change. Provide "section" + and "entry_text". +- amend: the block already has an entry for this issue, or for the + same user-visible change, and this PR extends or corrects it. + Provide "amend_gh" (the issue number of that entry) and "entry_text", + a full replacement entry that keeps the same :gh:`N` reference and + now covers both changes. +- skip: the block already fully describes this PR. Provide + "skip_reason". + +The script places the text and handles ordering, blank lines, badge +labels and the credits file. It rejects an insert whose issue already +has an entry, so pick amend in that case. + +ISSUE NUMBER SELECTION + +The entry should reference the GitHub ISSUE number when one exists, +not the pull request number. + +1. If the PR title or description references an issue such as + "Fixes #1234", "Closes #1234", "Refs #1234" or "#1234", use that + number. +2. If multiple issues are referenced, choose the primary one. +3. If no issue reference exists, fall back to the PR number. + +STYLE + +- Write concise entries (1-2 sentences max). +- Focus on the user-visible behavior change. +- Avoid implementation details unless relevant. +- Prefer imperative verbs: "fix", "add", "improve", "avoid", "detect". +- Do not repeat the PR title verbatim. +- Do not mention "PR" in the text. +- Wrap lines around ~79 characters. + +CLASSIFICATION + +Choose the section for an insert, from the vocabulary the changelog +uses (listed in the order sections appear in a release): + +- New APIs: a new function, argument or field, or an existing one now + working where it didn't. +- New platforms: support for a new OS, architecture or interpreter. +- API changes: an existing API changed, was deprecated or was removed. +- Performance: something got faster. +- Build and packaging: wheels, the sdist, what gets installed. +- Documentation: docs/ only. +- Internals: psutil's own machinery: CI, tests, scripts, debug output. + Nothing user-facing. +- Dropped support: a platform, OS or Python version is no longer + supported. +- Bug fixes: crashes, wrong results, leaks, build failures. + +PLATFORM TAGS + +If the change is platform specific, add tags immediately after the +issue reference, e.g.: + +:gh:`1234`, [Linux]: +:gh:`1234`, [macOS], [BSD]: + +Known tags: [Linux], [Windows], [macOS], [FreeBSD], [NetBSD], +[OpenBSD], [SunOS], [AIX], [BSD], [UNIX], [PyPy]. Use the specific OS +when one is named; [BSD] only when the change is about the BSDs as a +family, [UNIX] for shared POSIX behavior where no single OS fits. +Only add platform tags if the change clearly affects specific OSes. +The script checks every tag against the issue's labels on the bug +tracker and rejects the entry when a tag isn't backed by one. + +BADGE LABELS + +Do not write :label:`...` badges (critical, build-fail, memleak, +breaking). The script derives them from the issue's labels on the +tracker and adds them to the entry itself. + +RST FORMATTING + +Use Sphinx roles for psutil APIs: :func:`function_name`, +:meth:`Class.method`, :class:`ClassName`, :exc:`ExceptionName`. C +functions or identifiers use double backticks: ``function_name()``. + +ENTRY FORMAT + +An entry is a single bullet: + +- :gh:`ISSUE_NUMBER`: . + +Or with platform tags: + +- :gh:`ISSUE_NUMBER`, [Linux]: . + +Continuation lines are indented by two spaces. End with a period. + +EXAMPLES + +- :gh:`2708`, [macOS]: :meth:`Process.cmdline` and + :meth:`Process.environ` may fail with ``OSError: [Errno 0]``. They + now raise :exc:`AccessDenied` instead. + +- :gh:`2674`, [Windows]: :func:`disk_usage` could truncate values on + 32-bit systems for drives larger than 4GB. +""" + +SUBMIT_TOOL = { + "name": "submit", + "description": "Submit the changelog decision for this PR.", + "input_schema": { + "type": "object", + "additionalProperties": False, + "properties": { + "action": { + "type": "string", + "enum": ["insert", "amend", "skip"], + "description": ( + "insert: no entry for this change exists yet. amend:" + " the block already has an entry for this issue that" + " should be extended/corrected. skip: the block already" + " fully covers this PR." + ), + }, + "section": { + "type": ["string", "null"], + "enum": [ + "New APIs", + "New platforms", + "API changes", + "Performance", + "Build and packaging", + "Documentation", + "Internals", + "Dropped support", + "Bug fixes", + None, + ], + "description": "Required for insert.", + }, + "entry_text": { + "type": ["string", "null"], + "description": ( + "Full RST entry, wrapped at ~79 cols. Required for" + " insert/amend. For amend, the complete replacement" + " entry (must keep the same :gh:`N` reference)." + ), + }, + "amend_gh": { + "type": ["integer", "null"], + "description": ( + "Issue number of the existing entry to replace." + " Required for amend." + ), + }, + "skip_reason": { + "type": ["string", "null"], + "description": "One sentence. Required for skip.", + }, + }, + "required": [ + "action", + "section", + "entry_text", + "amend_gh", + "skip_reason", + ], + }, +} + + +def gh_request(path, accept="application/vnd.github+json"): + url = f"https://api.github.com{path}" + req = urllib.request.Request( + url, + headers={ + "Authorization": f"Bearer {TOKEN}", + "Accept": accept, + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as resp: + return resp.read() + except urllib.error.HTTPError as err: + # Surface GitHub's error body; a bare "HTTP Error 404" hides it. + body = err.read().decode("utf-8", errors="replace") + sys.exit(f"GitHub API {err.code} for {path}: {body}") + + +def fetch_pr_metadata(): + pr = json.loads(gh_request(f"/repos/{REPO}/pulls/{PR_NUMBER}")) + author = pr["user"]["login"] + # Fetch the user profile to get the full name. + user = json.loads(gh_request(f"/users/{author}")) + author_name = user.get("name") or author + return { + "number": pr["number"], + "title": pr["title"], + "body": pr.get("body") or "", + "author": author, + "author_name": author_name, + "head_sha": pr["head"]["sha"], + } + + +def fetch_pr_diff(): + return gh_request( + f"/repos/{REPO}/pulls/{PR_NUMBER}", + accept="application/vnd.github.v3.diff", + ).decode("utf-8", errors="replace") + + +def stale_reason(head_sha): + """Why the branch is too old to receive an entry, if it is. + We commit on top of the PR head, so an entry written against a + stale copy of the docs makes the PR unmergeable. + """ + data = json.loads(gh_request(f"/repos/{REPO}/compare/{head_sha}...master")) + files = [f["filename"] for f in data.get("files", [])] + stale = [f for f in (CHANGELOG_FILE, CREDITS_FILE) if f in files] + if stale: + return f"master has moved on: {' and '.join(stale)} changed" + if len(files) >= COMPARE_MAX_FILES: + return "master's diff is too big to tell whether the docs changed" + return None + + +def ask_claude(pr, diff, block): + prompt = PROMPT.format( + number=pr["number"], + title=pr["title"], + author=pr["author"], + author_name=pr["author_name"], + body=pr["body"], + diff=diff[:MAX_DIFF_CHARS], + block=block, + ) + import anthropic + + api_key = os.environ.get("ANTHROPIC_API_KEY", "").strip() + if not api_key: + sys.exit("ANTHROPIC_API_KEY is not set") + client = anthropic.Anthropic(api_key=api_key) + message = client.messages.create( + model=MODEL, + max_tokens=MAX_TOKENS, + thinking={"type": "adaptive"}, + output_config={"effort": "low"}, + tools=[SUBMIT_TOOL], + tool_choice={"type": "tool", "name": "submit"}, + messages=[{"role": "user", "content": prompt}], + ) + if message.stop_reason == "max_tokens": + sys.exit("Claude response was truncated (raise MAX_TOKENS)") + tool_use = next((b for b in message.content if b.type == "tool_use"), None) + if tool_use is None: + sys.exit( + f"Claude returned no tool call (stop_reason={message.stop_reason})" + ) + return tool_use.input + + +VERSION_RE = re.compile(r"^(\d+\.\d+\.\d+|X\.X\.X)\b") + +# The section vocabulary, in the order sections appear in a release. +SECTIONS = ( + "New APIs", + "New platforms", + "API changes", + "Performance", + "Build and packaging", + "Documentation", + "Internals", + "Dropped support", + "Bug fixes", +) + +# Badge labels: (tracker label, badge name), in the order badges +# appear in an entry and entries sort within a section. +BADGES = ( + ("critical", "critical"), + ("build-fail", "build-fail"), + ("memleak", "memleak"), + ("compatibility", "breaking"), +) + +# Platform tags accepted in an entry, with the tracker label that must +# back each one. +TAG2LABEL = { + "Linux": "linux", + "Windows": "windows", + "macOS": "macos", + "FreeBSD": "freebsd", + "NetBSD": "netbsd", + "OpenBSD": "openbsd", + "SunOS": "sunos", + "AIX": "aix", + "BSD": "bsd", + "UNIX": "unix", + "POSIX": "unix", + "PyPy": "pypy", +} +NAMED_BSD = {"freebsd", "netbsd", "openbsd"} +POSIXY = NAMED_BSD | {"linux", "macos", "sunos", "aix", "bsd", "unix"} + +ENTRY_PREFIX_RE = re.compile( + r"^- (?::gh:`\d+`, )*:gh:`\d+`((?:,? \[[^\]]+\])*)" +) + + +class ValidationError(Exception): + """The LLM decision doesn't match the file state; fail loud.""" + + +def find_dev_block(lines): + """Locate the top version block in a list of changelog lines. + + Returns (start, end, header, is_dev): start/end are line indices + (end points at the next version header or len(lines)); is_dev is + False when the top block is an already-released, dated version. + """ + start = next( + (i for i, ln in enumerate(lines) if VERSION_RE.match(ln)), None + ) + if start is None: + sys.exit(f"No version header found in {CHANGELOG_FILE}") + end = next( + ( + i + for i in range(start + 1, len(lines)) + if VERSION_RE.match(lines[i]) + ), + len(lines), + ) + header = lines[start] + is_dev = "(IN DEVELOPMENT)" in header or header.startswith("X.X.X") + return start, end, header, is_dev + + +def changelog_context(text): + """Return only the top version block, to feed to the LLM.""" + lines = text.splitlines() + start, end, _, _ = find_dev_block(lines) + return "\n".join(lines[start:end]).rstrip() + "\n" + + +def section_header_idx(block, section): + header = f"**{section}**" + return next( + (i for i, ln in enumerate(block) if ln.strip() == header), None + ) + + +def next_header_idx(block, from_idx): + """Index of the next ``**...**`` header after from_idx, else end.""" + return next( + ( + i + for i in range(from_idx + 1, len(block)) + if block[i].startswith("**") + ), + len(block), + ) + + +def section_rank(header_name): + """Canonical position of a section header, or None. + + Per-platform variants ("Bug fixes: Linux") rank as their base. + """ + base = header_name.split(":")[0].strip() + return SECTIONS.index(base) if base in SECTIONS else None + + +def create_section(block, section): + """Insert an empty ``**section**`` header keeping the canonical + section order. + """ + rank = SECTIONS.index(section) + new = [f"**{section}**", ""] + for i, ln in enumerate(block): + if ln.startswith("**") and ln.rstrip().endswith("**"): + got = section_rank(ln.strip().strip("*")) + if got is not None and got > rank: + return block[:i] + [*new, ""] + block[i:] + insert_at = len(block) + while insert_at > 0 and not block[insert_at - 1].strip(): + insert_at -= 1 + return block[:insert_at] + ["", *new] + block[insert_at:] + + +def entry_number(line): + m = re.match(r"- :gh:`(\d+)`", line) + return int(m.group(1)) if m else None + + +def entry_tier(line): + """Sort tier of an entry: badge-labelled ones first, in badge + order, plain ones last. + """ + for i, (_, name) in enumerate(BADGES): + if f":label:`{name}`" in line: + return i + return len(BADGES) + + +def entry_tags(entry_text): + """The [Platform] tags in an entry's prefix (not its prose).""" + m = ENTRY_PREFIX_RE.match(entry_text) + if not m: + return [] + return re.findall(r"\[([^\]]+)\]", m.group(1)) + + +def check_platform_tags(entry_text, issue_labels): + """Every [Tag] must be backed by a label on the issue, so the + changelog never asserts a platform the tracker doesn't know + about. The family tags are backed by any label of the family. + """ + have = set(issue_labels) + for tag in entry_tags(entry_text): + label = TAG2LABEL.get(tag) + if label is None: + raise ValidationError(f"unknown platform tag [{tag}]") + if label in have: + continue + if label == "bsd" and have & NAMED_BSD: + continue + if label == "unix" and have & POSIXY: + continue + raise ValidationError( + f"[{tag}] is not backed by a {label!r} label on the issue" + f" (it has: {sorted(have) or 'none'})" + ) + + +def rst_words(text): + """Split on spaces, except inside a `...` or ``...`` span.""" + words, cur, span = [], "", 0 + i = 0 + while i < len(text): + if text[i] == "`": + run = 1 + while i + run < len(text) and text[i + run] == "`": + run += 1 + cur += "`" * run + i += run + if span == 0: + span = run + elif run >= span: + span = 0 + continue + if text[i] == " " and span == 0: + if cur: + words.append(cur) + cur = "" + else: + cur += text[i] + i += 1 + if cur: + words.append(cur) + return words + + +def wrap_entry(flat, width=79): + """Wrap a one-line entry into '- ' + two-space continuations.""" + out, cur = [], "" + for word in rst_words(flat): + trial = f"{cur} {word}" if cur else (" " + word if out else word) + if len(trial) > width and cur: + out.append(cur) + cur = " " + word + else: + cur = trial + if cur: + out.append(cur) + return "\n".join(out) + + +def inject_labels(entry_text, issue_labels): + """Add the :label:`...` badges the issue's labels call for, then + rewrap. The model never writes badges itself. + """ + flat = " ".join(ln.strip() for ln in entry_text.splitlines()) + badges = [name for lab, name in BADGES if lab in issue_labels] + if badges: + m = ENTRY_PREFIX_RE.match(flat) + roles = ", ".join(f":label:`{b}`" for b in badges) + flat = f"{flat[: m.end()]}, {roles}{flat[m.end():]}" + return wrap_entry(flat) + + +def insert_in_section(block, section, entry_lines): + hdr = section_header_idx(block, section) + if hdr is None: + block = create_section(block, section) + hdr = section_header_idx(block, section) + boundary = next_header_idx(block, hdr) + # End of the section's content, before any trailing blank lines. + end = boundary + while end > hdr + 2 and not block[end - 1].strip(): + end -= 1 + # Start of the last contiguous run of entries. A blank line or a + # pseudo-header (e.g. "New APIs:") ends a run, so we stay out of + # earlier groups. + run_start = end + while run_start > hdr + 2: + if not block[run_start - 1].startswith(("- ", " ")): + break + run_start -= 1 + # Insert after the last entry that sorts before ours: labelled + # entries lead (critical, build-fail, memleak, breaking), plain + # ones follow, numerically within each tier. Robust to a leading + # out-of-order entry (we compare keys, not positions). + gh = entry_number(entry_lines[0]) + tier = entry_tier(entry_lines[0]) + insert_at = run_start + i = run_start + while i < end: + num = entry_number(block[i]) + j = i + 1 + while j < end and block[j].startswith(" "): # skip continuations + j += 1 + if block[i].startswith("- "): + t = entry_tier(block[i]) + if t < tier or ( + t == tier and num is not None and gh is not None and num < gh + ): + insert_at = j + i = j + return block[:insert_at] + list(entry_lines) + block[insert_at:] + + +def check_entry_ordered(text, gh): + """Verify the new :gh:`gh` entry respects its run's ordering: + labelled entries first in badge order, then plain ones, numeric + within each tier. Raises ValidationError otherwise. A blank line + or a pseudo-header ends a run, so an earlier group doesn't count. + """ + block = changelog_context(text).splitlines() + idx = next( + (i for i, ln in enumerate(block) if ln.startswith(f"- :gh:`{gh}`")), + None, + ) + if idx is None: + return + mine = (entry_tier(block[idx]), gh) + + def run_neighbor(indices): + for i in indices: + ln = block[i] + if ln.startswith("- "): + return (entry_tier(ln), entry_number(ln)) + if not ln.startswith(" "): # blank / pseudo-header + return None + return None + + def sorts_after(a, b): + if a[0] != b[0]: + return a[0] > b[0] + if a[1] is None or b[1] is None: + return False + return a[1] > b[1] + + prev = run_neighbor(range(idx - 1, -1, -1)) + nxt = run_neighbor(range(idx + 1, len(block))) + if prev is not None and sorts_after(prev, mine): + raise ValidationError( + f"changelog: :gh:`{gh}` is out of order (after {prev})" + ) + if nxt is not None and sorts_after(mine, nxt): + raise ValidationError( + f"changelog: :gh:`{gh}` is out of order (before {nxt})" + ) + + +def prepend_dev_block(lines, start): + header = "X.X.X (IN DEVELOPMENT)" + new = [header, "^" * len(header), ""] + lines = lines[:start] + new + lines[start:] + return lines, start, start + len(new) + + +def insert_entry(text, section, entry_text): + """Append an entry to a section, creating a dev block if needed.""" + lines = text.splitlines() + start, end, _, is_dev = find_dev_block(lines) + if not is_dev: + lines, start, end = prepend_dev_block(lines, start) + block = lines[start:end] + block = insert_in_section(block, section, entry_text.splitlines()) + lines[start:end] = block + return "\n".join(lines) + "\n" + + +def find_entry_span(block, gh): + """Return (start, end) of the entry for :gh:`gh` within block lines. + + The span covers the ``- :gh:`gh``` line plus its continuation + lines. Returns None if there's no such entry. + """ + prefix = f"- :gh:`{gh}`" + start = next( + (i for i, ln in enumerate(block) if ln.startswith(prefix)), None + ) + if start is None: + return None + end = start + 1 + while end < len(block): + ln = block[end] + if ( + ln.startswith(("- ", "**")) + or not ln.strip() + or VERSION_RE.match(ln) + ): + break + end += 1 + return start, end + + +def amend_entry(text, gh, entry_text): + """Replace the dev-block entry for :gh:`gh` with entry_text.""" + lines = text.splitlines() + start, end, _, _ = find_dev_block(lines) + block = lines[start:end] + prefix = f"- :gh:`{gh}`" + matches = [i for i, ln in enumerate(block) if ln.startswith(prefix)] + if len(matches) != 1: + raise ValidationError( + f"expected one entry for :gh:`{gh}` to amend, found {len(matches)}" + ) + span_start, span_end = find_entry_span(block, gh) + block[span_start:span_end] = entry_text.splitlines() + lines[start:end] = block + return "\n".join(lines) + "\n" + + +ENTRY_RE = re.compile( + r"^- :gh:`(\d+)`(?:,? \[[^\]]+\])*(?:, :label:`[a-z-]+`)*:\s" +) + + +def referenced_issues(title, body, pr_number): + """Issue numbers this PR may legitimately reference.""" + text = f"{title}\n{body}" + pat = r"#(\d+)|GH-(\d+)|/(?:issues|pull)/(\d+)" + refs = {int(n) for group in re.findall(pat, text) for n in group if n} + refs.add(pr_number) + return refs + + +def validate_entry_text(entry_text): + """Check an entry's RST shape; return its issue number.""" + entry_lines = entry_text.splitlines() + m = ENTRY_RE.match(entry_lines[0]) if entry_lines else None + if not m: + raise ValidationError( + f"entry must start with '- :gh:`N`...: ', got: {entry_text!r}" + ) + if not entry_text.rstrip().endswith("."): + raise ValidationError("entry must end with a period") + if entry_text.count("``") % 2 or entry_text.count("`") % 2: + raise ValidationError("entry has unbalanced backticks") + for ln in entry_lines[1:]: + if ln.startswith("- "): + raise ValidationError("entry must be a single bullet") + if ln.strip() and not ln.startswith(" "): + raise ValidationError("continuation lines must be indented") + return int(m.group(1)) + + +BUGFIX_FAMILY = { + "linux": "Linux", + "windows": "Windows", + "macos": "macOS", + "freebsd": "BSD", + "netbsd": "BSD", + "openbsd": "BSD", + "bsd": "BSD", + "sunos": "UNIX", + "aix": "UNIX", + "unix": "UNIX", +} + + +def bugfix_section(text, entry_text): + """When the dev block splits Bug fixes per platform (8.0.0 does), + route the entry by its platform tags. Otherwise plain "Bug fixes". + """ + block = changelog_context(text).splitlines() + subs = { + ln.strip().strip("*") + for ln in block + if ln.strip().startswith("**Bug fixes:") + } + if not subs: + return "Bug fixes" + labels = {TAG2LABEL[t] for t in entry_tags(entry_text)} + names = {BUGFIX_FAMILY.get(lab) for lab in labels} - {None} + name = names.pop() if len(names) == 1 else "cross-platform" + for candidate in (f"Bug fixes: {name}", "Bug fixes: cross-platform"): + if candidate in subs: + return candidate + return "Bug fixes" + + +def apply_changelog_decision(text, decision, allowed_issues): + """Validate and apply the LLM decision to the changelog text. + + Returns (new_text, status, gh) where status is "inserted", + "amended" or "skipped" and gh is the issue number touched (None on + skip). Raises ValidationError on any mismatch, so nothing is + written on bad input. + """ + action = decision["action"] + if action == "skip": + return text, "skipped", None + + entry_text = decision.get("entry_text") + if not entry_text: + raise ValidationError(f"{action} requires entry_text") + gh = validate_entry_text(entry_text) + _, _, _, is_dev = find_dev_block(text.splitlines()) + + if action == "insert": + # An insert must reference an issue the PR actually touches; an + # amend is authorized by the entry already existing in the block + # (checked in amend_entry), so the PR need not restate it. + if gh not in allowed_issues: + raise ValidationError( + f":gh:`{gh}` is not referenced by this PR" + f" (allowed: {sorted(allowed_issues)})" + ) + section = decision.get("section") + if section not in SECTIONS: + raise ValidationError(f"invalid section {section!r}") + if section == "Bug fixes": + section = bugfix_section(text, entry_text) + # Only dedup against a real in-development block; after a + # release the top block is history, and a new entry belongs in + # a fresh dev block regardless of what shipped. + if is_dev: + block_lines = changelog_context(text).splitlines() + if any(ln.startswith(f"- :gh:`{gh}`") for ln in block_lines): + raise ValidationError( + f"an entry for :gh:`{gh}` already exists; amend it instead" + ) + new_text = insert_entry(text, section, entry_text) + check_entry_ordered(new_text, gh) + return new_text, "inserted", gh + + if action == "amend": + if not is_dev: + raise ValidationError( + "top changelog block is released; cannot amend, insert instead" + ) + amend_gh = decision.get("amend_gh") + if amend_gh is None: + raise ValidationError("amend requires amend_gh") + if gh != amend_gh: + raise ValidationError( + f"amended entry must keep :gh:`{amend_gh}`, got :gh:`{gh}`" + ) + return amend_entry(text, amend_gh, entry_text), "amended", gh + + raise ValidationError(f"unknown action {action!r}") + + +def credit_line(author, author_name, gh): + if author_name and author_name != author: + who = f":user:`{author_name} <{author}>`" + else: + who = f":user:`{author}`" + return f"* {who} - :gh:`{gh}`" + + +def credits_sort_key(line): + s = line.strip() + # :user:`Name ` or :user:`handle` + m = re.match(r"\*\s+:user:`([^<`]+?)(?:\s*<[^>]+>)?`", s) + if m: + return m.group(1).strip().lower() + # legacy: * `Display Name`_ - ... + m = re.match(r"\*\s+`([^`]+)`_", s) + if m: + return m.group(1).strip().lower() + return s.lower() + + +def credits_handle(line): + m = re.search(r":user:`(?:[^<`]+<([^>]+)>|([^`<]+))`", line) + if not m: + return None + return (m.group(1) or m.group(2)).strip() + + +def legacy_name_to_handle(lines): + """Map a legacy ``Name``_ display name to its GitHub handle. + + Built from the ``.. _`Name`: https://github.com/`` target + definitions at the bottom of credits.rst. + """ + out = {} + pat = r"\.\. _`([^`]+)`:\s*https?://github\.com/([^/\s]+)" + for ln in lines: + m = re.match(pat, ln.strip()) + if m: + out[m.group(1).strip()] = m.group(2).strip() + return out + + +def line_handle(line, name_map): + """GitHub handle for a credits line (``:user:`` or legacy form).""" + handle = credits_handle(line) + if handle: + return handle + m = re.match(r"\*\s+`([^`]+)`_", line) + if m: + return name_map.get(m.group(1).strip()) + return None + + +def append_gh_to_line(line, gh): + ref = f":gh:`{gh}`" + # Insert before a trailing parenthetical note like "(wheels ...)". + m = re.search(r"\s+\([^)]*\)\s*$", line) + if m: + return f"{line[: m.start()]}, {ref}{line[m.start():]}" + return f"{line.rstrip()}, {ref}" + + +def apply_credit(text, author, author_name, gh, year): + """Add or extend the author's credits line for the given year. + + Returns (new_text, status) where status is "added", "appended" or + "skipped". @giampaolo is never credited. + """ + if author == "giampaolo": + return text, "skipped" + lines = text.splitlines() + year_re = re.compile(r"^\d{4}$") + section_idx = next( + ( + i + for i, ln in enumerate(lines) + if ln.strip() == "Code contributors by year" + ), + None, + ) + if section_idx is None: + sys.exit(f"'Code contributors by year' not found in {CREDITS_FILE}") + + year_idx = next( + ( + i + for i in range(section_idx, len(lines)) + if lines[i].strip() == year + ), + None, + ) + if year_idx is None: + first_year = next( + ( + i + for i in range(section_idx, len(lines)) + if year_re.match(lines[i].strip()) + ), + len(lines), + ) + lines[first_year:first_year] = [ + year, + "~" * len(year), + "", + credit_line(author, author_name, gh), + "", + ] + return "\n".join(lines) + "\n", "added" + + year_end = next( + ( + i + for i in range(year_idx + 2, len(lines)) + if year_re.match(lines[i].strip()) + ), + len(lines), + ) + name_map = legacy_name_to_handle(lines) + for i in range(year_idx + 2, year_end): + ln = lines[i] + if ln.startswith("* ") and line_handle(ln, name_map) == author: + # A credit may wrap onto 2-space-indented continuation + # lines; treat the whole thing as one logical entry. + j = i + 1 + while j < year_end and lines[j].startswith(" "): + j += 1 + entry = lines[i:j] + if any(f":gh:`{gh}`" in ln for ln in entry): + return text, "skipped" + appended = append_gh_to_line(entry[-1], gh) + if len(appended) <= 79: + entry[-1] = appended + else: + entry[-1] = entry[-1].rstrip() + "," + entry.append(f" :gh:`{gh}`") + lines[i:j] = entry + result = resort_credits_year("\n".join(lines) + "\n", year) + return result, "appended" + + new_line = credit_line(author, author_name, gh) + key = credits_sort_key(new_line) + insert_idx = year_end + for i in range(year_idx + 2, year_end): + if lines[i].startswith("* ") and credits_sort_key(lines[i]) > key: + insert_idx = i + break + min_idx = year_idx + 3 + while insert_idx > min_idx and not lines[insert_idx - 1].strip(): + insert_idx -= 1 + lines.insert(insert_idx, new_line) + result = resort_credits_year("\n".join(lines) + "\n", year) + return result, "added" + + +def resort_credits_year(text, year): + """Sort the year's credit entries alphabetically, keeping each + entry's wrapped continuation lines with it. + """ + lines = text.splitlines() + year_re = re.compile(r"^\d{4}$") + section_idx = next( + ( + i + for i, ln in enumerate(lines) + if ln.strip() == "Code contributors by year" + ), + None, + ) + if section_idx is None: + return text + year_idx = next( + ( + i + for i in range(section_idx, len(lines)) + if lines[i].strip() == year + ), + None, + ) + if year_idx is None: + return text + start = year_idx + 2 + while start < len(lines) and not lines[start].strip(): + start += 1 + end = next( + ( + i + for i in range(year_idx + 2, len(lines)) + if year_re.match(lines[i].strip()) + ), + len(lines), + ) + while end > start and not lines[end - 1].strip(): + end -= 1 + entries = [] + for ln in lines[start:end]: + if ln.startswith("* ") or not entries: + entries.append([ln]) + else: + entries[-1].append(ln) + entries.sort(key=lambda e: credits_sort_key(e[0])) + lines[start:end] = [ln for entry in entries for ln in entry] + return "\n".join(lines) + "\n" + + +def write_file(path, body): + """Write a PR comment body for the workflow to post later. + + Success and validation-error bodies go to different files so the + workflow's failure step never posts a stale "entry added" body when + a later step (e.g. the push) is what failed. + """ + print(body) + if path: + with open(path, "w") as f: + f.write(body) + + +def parse_cli(): + global PR_NUMBER, REPO, TOKEN, COMMENT_FILE, ERROR_FILE + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--pr-number", type=int, required=True) + p.add_argument( + "--repo", type=str, required=True, help="e.g. giampaolo/psutil" + ) + p.add_argument("--token", type=str, required=True, help="GitHub token") + p.add_argument( + "--comment-file", + type=str, + default=None, + help="On success, write the PR comment body here", + ) + p.add_argument( + "--error-file", + type=str, + default=None, + help="On a validation failure, write the error comment here", + ) + args = p.parse_args() + PR_NUMBER = args.pr_number + REPO = args.repo + TOKEN = args.token + COMMENT_FILE = args.comment_file + ERROR_FILE = args.error_file + + +def build_comment(cl_status, cr_status, decision, gh, author): + """Compose the PR comment strictly from what the script did.""" + entry = decision.get("entry_text") or "" + block = f"```rst\n{entry}\n```" + if cl_status == "inserted": + section = decision["section"] + header = f"`{CHANGELOG_FILE}` entry added under **{section}**:" + parts = [f"{header}\n\n{block}"] + elif cl_status == "amended": + header = f"`{CHANGELOG_FILE}` entry for :gh:`{gh}` amended:" + parts = [f"{header}\n\n{block}"] + else: + reason = decision.get("skip_reason") or "already covered" + parts = [f"`{CHANGELOG_FILE}` **not** modified (skipped): {reason}"] + + if cr_status == "added": + parts.append(f"`{CREDITS_FILE}`: credited @{author} for :gh:`{gh}`.") + elif cr_status == "appended": + parts.append( + f"`{CREDITS_FILE}`: appended :gh:`{gh}` to @{author}'s line." + ) + return "\n\n".join(parts) + + +def fetch_issue_labels(number): + raw = json.loads(gh_request(f"/repos/{REPO}/issues/{number}")) + return sorted(x["name"] for x in raw["labels"]) + + +def run_decision(pr, decision, year=None, get_labels=fetch_issue_labels): + """Apply the decision to both files; return (cl_status, cr_status, gh).""" + allowed = referenced_issues(pr["title"], pr["body"], pr["number"]) + entry = decision.get("entry_text") + if decision.get("action") != "skip" and entry: + if ":label:`" in entry: + raise ValidationError( + "entry must not contain :label: badges; they are" + " derived from the issue's labels" + ) + labels = get_labels(validate_entry_text(entry)) + check_platform_tags(entry, labels) + # In place, so the PR comment shows what was really written. + decision["entry_text"] = inject_labels(entry, labels) + with open(CHANGELOG_FILE) as f: + changelog = f.read() + new_changelog, cl_status, gh = apply_changelog_decision( + changelog, decision, allowed + ) + # A no-op amend (identical text) changes nothing; report it as a + # skip so the comment doesn't claim an edit that never happened. + if cl_status != "skipped" and new_changelog == changelog: + cl_status, gh = "skipped", None + if cl_status != "skipped": + with open(CHANGELOG_FILE, "w") as f: + f.write(new_changelog) + print(f"Changelog: {cl_status}") + + cr_status = "skipped" + if gh is not None: + year = year or str(datetime.date.today().year) + with open(CREDITS_FILE) as f: + credits = f.read() + new_credits, cr_status = apply_credit( + credits, pr["author"], pr["author_name"], gh, year + ) + if cr_status != "skipped": + with open(CREDITS_FILE, "w") as f: + f.write(new_credits) + print(f"Credits: {cr_status}") + return cl_status, cr_status, gh + + +def main(): + parse_cli() + print(f"Fetching PR #{PR_NUMBER} from {REPO}...") + pr = fetch_pr_metadata() + stale = stale_reason(pr["head_sha"]) + if stale: + write_file( + ERROR_FILE, + f"âš ï¸ The /changelog bot did nothing: {stale}. Committing an" + " entry now would make this PR unmergeable. Click **Update" + " branch**, then comment `/changelog` again.", + ) + sys.exit(f"stale branch: {stale}") + diff = fetch_pr_diff() + with open(CHANGELOG_FILE) as f: + block = changelog_context(f.read()) + print("Asking Claude for changelog decision...") + decision = ask_claude(pr, diff, block) + print(f"Decision: {decision}") + try: + cl_status, cr_status, gh = run_decision(pr, decision) + except ValidationError as err: + blob = json.dumps(decision, indent=2) + write_file( + ERROR_FILE, + f"âš ï¸ The /changelog bot could not apply the entry: {err}\n\n" + f"Model output:\n\n```json\n{blob}\n```", + ) + sys.exit(f"validation failed: {err}") + write_file( + COMMENT_FILE, + build_comment(cl_status, cr_status, decision, gh, pr["author"]), + ) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/changelog_bot.yml b/.github/workflows/changelog_bot.yml new file mode 100644 index 0000000000..3a535e6647 --- /dev/null +++ b/.github/workflows/changelog_bot.yml @@ -0,0 +1,145 @@ +name: changelog-bot + +on: + issue_comment: + types: [created] + +permissions: + contents: write + pull-requests: write + +jobs: + changelog: + if: > + contains(github.event.comment.body, '/changelog') && + github.event.issue.pull_request != null + runs-on: ubuntu-latest + + # Don't let two /changelog comments on the same PR race the + # checkout/push. Queue the second one instead of cancelling. + concurrency: + group: changelog-${{ github.event.issue.number }} + cancel-in-progress: false + + steps: + - name: Check commenter permissions + id: check-perms + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMMENTER: ${{ github.event.comment.user.login }} + BODY: ${{ github.event.comment.body }} + run: | + # The `if:` above can't trim, and the browser sends the + # command as "\r\n/changelog\r\n\r\n". + if [ "$(printf '%s' "$BODY" | tr -d '[:space:]')" != "/changelog" ] + then + echo "Not a /changelog command" + exit 1 + fi + PERMISSION=$(gh api \ + "repos/${{ github.repository }}/collaborators/$COMMENTER/permission" \ + --jq '.permission') + echo "permission=$PERMISSION" + if [[ "$PERMISSION" != "write" && "$PERMISSION" != "admin" ]]; then + echo "User $COMMENTER does not have write/admin permission" + exit 1 + fi + + - name: Get PR info + id: pr-info + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.issue.number }} + run: | + PR_JSON=$(gh pr view "$PR_NUMBER" \ + --repo "${{ github.repository }}" \ + --json headRefName,headRepository,headRepositoryOwner,headRefOid) + HEAD_BRANCH=$(echo "$PR_JSON" | jq -r '.headRefName') + HEAD_OWNER=$(echo "$PR_JSON" | jq -r '.headRepositoryOwner.login') + HEAD_REPO=$(echo "$PR_JSON" | jq -r '.headRepository.name') + HEAD_SHA=$(echo "$PR_JSON" | jq -r '.headRefOid') + echo "branch=$HEAD_BRANCH" >> "$GITHUB_OUTPUT" + echo "head_repo=$HEAD_OWNER/$HEAD_REPO" >> "$GITHUB_OUTPUT" + echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" + + - name: Checkout PR branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ steps.pr-info.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: pip install anthropic rstwrap + + - name: Run changelog bot + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + PR_NUMBER: ${{ github.event.issue.number }} + run: | + # Run the trusted script from master, not the PR's copy, so a + # malicious PR can't get arbitrary code executed on the runner. + git fetch --depth 1 origin master + git show origin/master:.github/workflows/changelog_bot.py > /tmp/changelog_bot.py + python /tmp/changelog_bot.py \ + --pr-number "$PR_NUMBER" \ + --repo "${{ github.repository }}" \ + --token "${{ secrets.GITHUB_TOKEN }}" \ + --comment-file "$RUNNER_TEMP/changelog_comment.md" \ + --error-file "$RUNNER_TEMP/changelog_error.md" + + - name: Commit and push if changelog changed + id: push + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.issue.number }} + # Pass fork-controlled values through env, never interpolate + # ${{ }} into the shell: a branch named "x$(cmd)" would + # otherwise run as a command. + HEAD_REPO: ${{ steps.pr-info.outputs.head_repo }} + HEAD_BRANCH: ${{ steps.pr-info.outputs.branch }} + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + rstwrap docs/changelog.rst docs/credits.rst + git add docs/changelog.rst docs/credits.rst + if git diff --cached --quiet; then + echo "No changes, skipping commit" + else + git commit --no-verify -m "Update changelog for PR #$PR_NUMBER" + PUSH_URL="https://x-access-token:${GH_TOKEN}@github.com/${HEAD_REPO}.git" + git push "$PUSH_URL" "HEAD:refs/heads/${HEAD_BRANCH}" + fi + + - name: Post confirmation comment + # Best-effort: a comment hiccup must not mislabel a landed + # push as a failure. + if: success() + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.issue.number }} + run: | + gh pr comment "$PR_NUMBER" \ + --repo "${{ github.repository }}" \ + --body-file "$RUNNER_TEMP/changelog_comment.md" + + - name: Report failure on the PR + if: failure() && steps.check-perms.outcome == 'success' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.issue.number }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + # A validation error leaves a specific message; anything else + # (e.g. a failed push) only gets the generic pointer, never a + # stale success body. + ERROR_FILE="$RUNNER_TEMP/changelog_error.md" + if [ -s "$ERROR_FILE" ]; then + gh pr comment "$PR_NUMBER" \ + --repo "${{ github.repository }}" \ + --body-file "$ERROR_FILE" + else + gh pr comment "$PR_NUMBER" \ + --repo "${{ github.repository }}" \ + --body "âš ï¸ The /changelog bot failed. See the run log: $RUN_URL" + fi diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000..26337add26 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,130 @@ +# Builds and tests the docs, then deploys them to GitHub Pages. +# +# Flow: sanity tests -> build -> deploy -> live-site tests. Single +# version, served at the site root (psutil.io). Nothing is committed: +# the built site is uploaded as an artifact and served directly (no +# gh-pages branch, no separate repo). +# +# Repo setup: Settings -> Pages -> Source = "GitHub Actions", and set +# the custom domain to psutil.io. +# +# Prior art if we ever add versioning. Preferred path is the Salt +# model: keep this artifact method and single repo, but fan the build +# out over master + the last few supported release tags (master -> /, +# each tag -> /vX.Y/) and rebuild that set every run. No separate repo, +# no committed HTML, EOL versions drop off. Root stays master so the +# blog keeps publishing; the /vX.Y/ dirs are frozen API snapshots. A +# JS flyout reads a generated versions.json. +# Salt (this model, in production): +# https://github.com/saltstack/builddocs/blob/main/.github/workflows/gh-pages-builddocs.yml +# NumPy (heavier: separate repo, one committed dir per release, by +# hand) - only for a permanent archive of every past release: +# https://github.com/numpy/numpy.org/blob/main/.github/workflows/gh-pages.yml +# https://github.com/numpy/doc + +on: + push: + branches: [master] + paths: &docs_paths + - "docs/**" + - ".github/workflows/docs.yml" + - "pyproject.toml" + pull_request: + paths: *docs_paths + workflow_dispatch: + +name: docs + +concurrency: + group: docs-deploy + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 # full history for sphinx-last-updated-by-git + + - uses: actions/setup-python@v7 + with: + python-version: 3.x + + - name: Install dependencies + run: | + make install-pydeps-docs + make install-pydeps-lint + make install-pydeps-test + pip install . # codeautolink imports psutil + + - name: Lint rst + run: make lint-rst + + - name: Doc sanity tests + run: make -C docs test + + - name: Refresh adoption stats + # Change is not committed to GIT, but ends up on the live site. + continue-on-error: true + run: python3 scripts/internal/docs/refresh_adoption_stats.py + + - name: Build HTML + run: make -C docs html + + - name: Build past doc releases + run: make -C docs versions + + - name: Add deploy marker + run: echo "${{ github.sha }}" > docs/_build/html/build-sha.txt + + - uses: actions/upload-pages-artifact@v5 + with: + path: docs/_build/html + + deploy: + needs: build + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v5 + + test-online: + needs: deploy + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: 3.x + - name: Install dependencies + run: | + make install-pydeps-test + make install-pydeps-docs + + - name: Wait for the live site to serve this commit + # deploy-pages reports "live" before the CDN edge catches up, + # so poll build-sha.txt (cache-busted) until it matches. + run: | + for i in $(seq 1 60); do + live=$(curl -fsS "https://psutil.io/build-sha.txt?cb=$i" || true) + if [ "$live" = "${{ github.sha }}" ]; then + echo "live site is serving ${{ github.sha }}" + exit 0 + fi + echo "waiting for CDN propagation ($i)..." + sleep 5 + done + echo "timed out waiting for the deploy to go live" + exit 1 + + - name: Live-site tests + run: make -C docs test-online-doc diff --git a/.github/workflows/sunos.yml b/.github/workflows/sunos.yml new file mode 100644 index 0000000000..84b4af4413 --- /dev/null +++ b/.github/workflows/sunos.yml @@ -0,0 +1,49 @@ +# Execute tests on SunOS +# https://github.com/vmactions/solaris-vm + +name: sunos +on: + workflow_dispatch: + push: + # only run this job if the following files are modified + paths: &sunos_paths + - ".github/workflows/sunos.yml" + - "Makefile" + - "psutil/__init__.py" + - "psutil/_common.py" + - "psutil/_ntuples.py" + - "psutil/_psposix.py" + - "psutil/_pssunos.py" + - "psutil/_psutil_sunos.c" + - "psutil/arch/all/**" + - "psutil/arch/posix/**" + - "psutil/arch/sunos/**" + - "pyproject.toml" + - "scripts/internal/install-pydeps.sh" + - "scripts/internal/install-sysdeps.sh" + - "setup.py" + pull_request: + paths: *sunos_paths +concurrency: + group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.sha || '' }} + cancel-in-progress: true +jobs: + test: + # Skip same-repo PR runs: the push event already covers them. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + - name: Run tests + id: test + uses: vmactions/solaris-vm@v1 + with: + release: "11.4-gcc" + usesh: true + run: | + set -x + pkg install --accept developer/build/gnu-make || true + gmake build + gmake install-pydeps-test + gmake test-memleaks-parallel diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000000..417669d762 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,90 @@ +# Runs CI tests against a source build on the following platforms: +# * Linux +# * macOS +# * Windows +# +# Wheels are built by wheels.yml. +# +# Useful URLs: +# * https://github.com/actions/cache +# * https://github.com/actions/checkout +# * https://github.com/actions/setup-python + +on: + workflow_dispatch: + push: + paths-ignore: + - "docs/**" + pull_request: + paths-ignore: + - "docs/**" +name: tests +concurrency: + # Cancel run if a new one starts, but don't interrupt all jobs on the first + # failure. + group: test-${{ github.ref }} + cancel-in-progress: true +jobs: + + # Run tests on Linux, macOS, Windows + tests: + name: "${{ matrix.osname }} ${{ matrix.arch }} (py${{ matrix.python }})" + # Skip same-repo PR runs: the push event already covers them. + if: &skip_same_repo_pr github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, osname: linux, arch: x86_64, python: "3.14", memleaks: true } + - { os: ubuntu-24.04-arm, osname: linux, arch: aarch64, python: "3.8" } + - { os: ubuntu-latest, osname: linux, arch: x86_64, python: "3.14t" } + - { os: macos-15, osname: macos, arch: arm64, python: "3.14", memleaks: true } + - { os: macos-15-intel, osname: macos, arch: x86_64, python: "3.13" } # py 3.9 install is too slow + - { os: windows-2025, osname: win, arch: AMD64, python: "3.10", memleaks: true } + - { os: windows-11-arm, osname: win, arch: ARM64, python: "3.14" } + steps: + - uses: actions/checkout@v7 + + - name: Install Python + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python }} + + # setup-python always provides "python"; "python3" is missing on + # Windows, and the Makefile defaults to it. + - name: Run tests + shell: bash + run: make PYTHON=python ci-test + + - name: Run memleak tests + if: matrix.memleaks + shell: bash + run: make PYTHON=python test-memleaks-parallel + + # Run linters + bots tests + linters: + if: *skip_same_repo_pr + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: 3.x + # Cache dprint binary + plugins, which would otherwise be recompiled + # on every run (~5s). + - name: Cache dprint + uses: actions/cache@v6 + with: + path: | + ~/.dprint + ~/.cache/dprint + key: dprint-${{ hashFiles('.dprint.jsonc') }} + - name: "Run linters" + run: | + make ci-lint + - name: "Run bot tests" + run: | + ./scripts/internal/install-pydeps.sh pytest + make test-bots diff --git a/.github/workflows/tests/test_changelog_bot.py b/.github/workflows/tests/test_changelog_bot.py new file mode 100644 index 0000000000..39e59bac32 --- /dev/null +++ b/.github/workflows/tests/test_changelog_bot.py @@ -0,0 +1,1094 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for the /changelog bot (.github/workflows/changelog_bot.py). + +The bot edits docs/changelog.rst and docs/credits.rst based on an LLM +decision. These tests exercise the deterministic file surgery and the +validation gate against small RST fixtures; no network or API is used. +""" + +import importlib.util +import io +import pathlib + +import pytest + +BOT_PATH = pathlib.Path(__file__).parent.parent / "changelog_bot.py" + + +def import_module_by_path(path): + spec = importlib.util.spec_from_file_location(path.stem, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +cb = import_module_by_path(BOT_PATH) + +CHANGELOG = """\ +Changelog +========= + +8.0.0 (IN DEVELOPMENT) +^^^^^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`100`: new API one. + +**Bug fixes** + +- :gh:`200`: bug one. +- :gh:`201`, [Linux]: bug two spanning + two physical lines. + +7.2.2 — 2026-01-28 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`50`: old released bug. +""" + + +class TestContext: + def test_returns_only_top_block(self): + block = cb.changelog_context(CHANGELOG) + assert block.startswith("8.0.0 (IN DEVELOPMENT)") + assert ":gh:`200`" in block + # The released block below must not be included. + assert "7.2.2" not in block + assert ":gh:`50`" not in block + + +class TestInsert: + def _lines(self, text): + return text.splitlines() + + def test_appends_to_end_of_section(self): + out = cb.insert_entry(CHANGELOG, "Bug fixes", "- :gh:`300`: new bug.") + lines = self._lines(out) + idx = lines.index("- :gh:`300`: new bug.") + # It lands after the existing bug entries. + assert lines.index("- :gh:`200`: bug one.") < idx + + def test_does_not_glue_to_next_header(self): + # The regression: a new entry must not abut the next version + # header with no blank line in between. + out = cb.insert_entry(CHANGELOG, "Bug fixes", "- :gh:`300`: new bug.") + lines = self._lines(out) + idx = lines.index("- :gh:`300`: new bug.") + assert lines[idx + 1] == "" + assert lines[idx + 2].startswith("7.2.2") + + def test_inserts_into_correct_section(self): + out = cb.insert_entry(CHANGELOG, "New APIs", "- :gh:`300`: new api.") + lines = self._lines(out) + api = lines.index("**New APIs**") + bug = lines.index("**Bug fixes**") + idx = lines.index("- :gh:`300`: new api.") + assert api < idx < bug + + def test_inserts_in_sorted_position(self): + # 150 < 200, so it must come before it (#123 before #124). + out = cb.insert_entry(CHANGELOG, "Bug fixes", "- :gh:`150`: early.") + lines = self._lines(out) + assert lines.index("- :gh:`150`: early.") < lines.index( + "- :gh:`200`: bug one." + ) + + def test_inserts_into_last_run_of_grouped_section(self): + # A section split into pseudo-header groups: a new + # entry goes into the last run, sorted, never into an earlier + # group even if its number would sort there. + text = """\ +8.0.0 (IN DEVELOPMENT) +^^^^^^^^^^^^^^^^^^^^^^ + +**Performance** + +New APIs: + +- :gh:`100`: api one. +- :gh:`300`: api three. + +Others: + +- :gh:`200`: other two. +- :gh:`400`: other four. +""" + out = cb.insert_entry(text, "Performance", "- :gh:`250`: new misc.") + lines = self._lines(out) + new = lines.index("- :gh:`250`: new misc.") + # In the Others run (after that header), not New APIs. + assert lines.index("Others:") < new + assert lines.index("- :gh:`200`: other two.") < new + assert new < lines.index("- :gh:`400`: other four.") + + def test_multiline_entry_kept_together(self): + entry = "- :gh:`300`: a long entry that wraps onto\n a second line." + out = cb.insert_entry(CHANGELOG, "Bug fixes", entry) + lines = self._lines(out) + idx = lines.index("- :gh:`300`: a long entry that wraps onto") + assert lines[idx + 1] == " a second line." + assert lines[idx + 2] == "" + + def test_creates_missing_section_in_canonical_order(self): + text = """\ +8.0.0 (IN DEVELOPMENT) +^^^^^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`100`: api. +""" + out = cb.insert_entry(text, "Bug fixes", "- :gh:`300`: new bug.") + lines = self._lines(out) + assert lines.index("**New APIs**") < lines.index("**Bug fixes**") + assert lines.index("**Bug fixes**") < lines.index( + "- :gh:`300`: new bug." + ) + + def test_created_section_has_blank_line_before_first_entry(self): + # Regression: a freshly created section must not glue its first + # entry to the header (renders as run-on prose, silently). + text = """\ +8.0.0 (IN DEVELOPMENT) +^^^^^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`100`: api. +""" + out = cb.insert_entry(text, "Bug fixes", "- :gh:`300`: new bug.") + lines = self._lines(out) + hdr = lines.index("**Bug fixes**") + assert lines[hdr + 1] == "" + assert lines[hdr + 2] == "- :gh:`300`: new bug." + + def test_created_mid_order_section_has_blank_line_before_entry(self): + text = """\ +8.0.0 (IN DEVELOPMENT) +^^^^^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`200`: bug. +""" + out = cb.insert_entry(text, "Performance", "- :gh:`300`: faster.") + lines = self._lines(out) + hdr = lines.index("**Performance**") + assert lines[hdr + 1] == "" + assert lines[hdr + 2] == "- :gh:`300`: faster." + + def test_creates_missing_section_before_later_one(self): + text = """\ +8.0.0 (IN DEVELOPMENT) +^^^^^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`200`: bug. +""" + out = cb.insert_entry(text, "Performance", "- :gh:`300`: faster.") + lines = self._lines(out) + assert lines.index("**Performance**") < lines.index("**Bug fixes**") + + +class TestLabels: + def test_inject_in_badge_order_and_wrapped(self): + entry = ( + "- :gh:`300`, [NetBSD]: a fairly long entry describing a" + " double free\n in :func:`swap_memory` on some systems." + ) + out = cb.inject_labels(entry, ["bug", "critical", "memleak", "netbsd"]) + first = out.splitlines()[0] + assert first.startswith( + "- :gh:`300`, [NetBSD], :label:`critical`, :label:`memleak`:" + ) + assert all(len(ln) <= 79 for ln in out.splitlines()) + + def test_compatibility_becomes_breaking(self): + out = cb.inject_labels( + "- :gh:`300`: renamed.", ["compatibility", "windows"] + ) + assert out.startswith("- :gh:`300`, :label:`breaking`: renamed.") + + def test_no_labels_no_badges(self): + out = cb.inject_labels("- :gh:`300`: plain.", ["bug", "linux"]) + assert ":label:" not in out + + def test_wrap_never_splits_inside_roles(self): + entry = ( + "- :gh:`300`: fix by" + " :user:`Somebody With A Very Long Name ` plus" + " ``a b c`` and more words to push this over the limit for" + " sure." + ) + out = cb.inject_labels(entry, []) + flat = " ".join(ln.strip() for ln in out.splitlines()) + assert ":user:`Somebody With A Very Long Name `" in flat + assert "``a b c``" in flat + + +class TestPlatformTags: + def test_backed_tag_passes(self): + cb.check_platform_tags("- :gh:`1`, [Linux]: x.", ["bug", "linux"]) + + def test_family_tag_backed_by_member(self): + cb.check_platform_tags("- :gh:`1`, [BSD]: x.", ["freebsd"]) + cb.check_platform_tags("- :gh:`1`, [UNIX]: x.", ["sunos"]) + + def test_unbacked_tag_rejected(self): + with pytest.raises(cb.ValidationError): + cb.check_platform_tags("- :gh:`1`, [Windows]: x.", ["linux"]) + + def test_brackets_in_prose_are_not_tags(self): + cb.check_platform_tags( + "- :gh:`1`: raises ``[Errno 2]`` sometimes.", [] + ) + + +class TestTierOrdering: + def test_labelled_entry_inserted_before_plain_ones(self): + entry = "- :gh:`300`, :label:`critical`: segfault." + out = cb.insert_entry(CHANGELOG, "Bug fixes", entry) + lines = out.splitlines() + assert lines.index(entry) < lines.index("- :gh:`200`: bug one.") + + def test_plain_entry_lands_after_labelled_run(self): + text = """\ +8.0.0 (IN DEVELOPMENT) +^^^^^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`500`, :label:`critical`: boom. +- :gh:`100`: plain. +""" + out = cb.insert_entry(text, "Bug fixes", "- :gh:`300`: also plain.") + lines = out.splitlines() + assert lines.index("- :gh:`100`: plain.") < lines.index( + "- :gh:`300`: also plain." + ) + assert lines.index( + "- :gh:`500`, :label:`critical`: boom." + ) < lines.index("- :gh:`100`: plain.") + + +class TestBugfixRouting: + SPLIT = """\ +Changelog +========= + +8.0.0 (IN DEVELOPMENT) +^^^^^^^^^^^^^^^^^^^^^^ + +**Bug fixes: cross-platform** + +- :gh:`10`: everywhere. + +**Bug fixes: Windows** + +- :gh:`20`, [Windows]: win bug. + +**Bug fixes: BSD** + +- :gh:`30`, [NetBSD]: bsd bug. +""" + + def test_routes_by_platform_tag(self): + assert ( + cb.bugfix_section(self.SPLIT, "- :gh:`40`, [Windows]: x.") + == "Bug fixes: Windows" + ) + assert ( + cb.bugfix_section(self.SPLIT, "- :gh:`40`, [FreeBSD]: x.") + == "Bug fixes: BSD" + ) + + def test_untagged_or_mixed_goes_cross_platform(self): + assert ( + cb.bugfix_section(self.SPLIT, "- :gh:`40`: x.") + == "Bug fixes: cross-platform" + ) + assert ( + cb.bugfix_section(self.SPLIT, "- :gh:`40`, [Linux], [SunOS]: x.") + == "Bug fixes: cross-platform" + ) + + def test_plain_block_stays_plain(self): + assert ( + cb.bugfix_section(CHANGELOG, "- :gh:`40`, [Windows]: x.") + == "Bug fixes" + ) + + +class TestOrderCheck: + ORDERED = """\ +8.0.0 (IN DEVELOPMENT) +^^^^^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`100`: a. +- :gh:`200`: b. +""" + + def test_passes_for_sorted_run(self): + cb.check_entry_ordered(self.ORDERED, 200) + + def test_raises_when_placed_after_higher_number(self): + text = self.ORDERED.replace("- :gh:`100`", "- :gh:`300`") + with pytest.raises(cb.ValidationError): + cb.check_entry_ordered(text, 200) + + def test_stops_at_group_boundary(self): + # A pseudo-header ends the run, so a higher number in an earlier + # group doesn't count as "before" this entry. + text = """\ +8.0.0 (IN DEVELOPMENT) +^^^^^^^^^^^^^^^^^^^^^^ + +**Enhancements** + +New APIs: + +- :gh:`300`: api. + +Others: + +- :gh:`200`: other. +""" + cb.check_entry_ordered(text, 200) + + +class TestAmend: + def _lines(self, text): + return text.splitlines() + + def test_replaces_single_line_entry(self): + out = cb.amend_entry( + CHANGELOG, 200, "- :gh:`200`: bug one, now amended." + ) + lines = self._lines(out) + assert "- :gh:`200`: bug one, now amended." in lines + assert "- :gh:`200`: bug one." not in lines + # Siblings untouched. + assert "- :gh:`201`, [Linux]: bug two spanning" in lines + + def test_replaces_multiline_entry_span(self): + out = cb.amend_entry(CHANGELOG, 201, "- :gh:`201`: collapsed.") + lines = self._lines(out) + assert "- :gh:`201`: collapsed." in lines + # Both old physical lines of the 201 entry are gone. + assert "- :gh:`201`, [Linux]: bug two spanning" not in lines + assert " two physical lines." not in lines + + def test_raises_when_entry_absent(self): + with pytest.raises(cb.ValidationError): + cb.amend_entry(CHANGELOG, 999, "- :gh:`999`: nope.") + + def test_does_not_match_released_block_entry(self): + # :gh:`50` only exists in the released 7.2.2 block. + with pytest.raises(cb.ValidationError): + cb.amend_entry(CHANGELOG, 50, "- :gh:`50`: nope.") + + +CREDITS = """\ +Code contributors +================= + +Code contributors by year +------------------------- + +2026 +~~~~ + +* `Amaan Qureshi`_ - :gh:`2770` +* :user:`Gabriel Changamire ` - :gh:`2809` +* :user:`Tobias Klauser ` - :gh:`2711` + +2025 +~~~~ + +* `Ben Peddell`_ - :gh:`2495`, :gh:`2568` +""" + + +class TestCreditLine: + def test_full_name_form(self): + line = cb.credit_line("hansonw", "Hanson Wang", 2809) + assert line == "* :user:`Hanson Wang ` - :gh:`2809`" + + def test_handle_only_form(self): + # author_name falls back to the handle when no full name is set. + line = cb.credit_line("someuser", "someuser", 2710) + assert line == "* :user:`someuser` - :gh:`2710`" + + +class TestCredits: + def _lines(self, text): + return text.splitlines() + + def test_adds_new_contributor_sorted(self): + out, status = cb.apply_credit( + CREDITS, "newuser", "New User", 300, "2026" + ) + assert status == "added" + lines = self._lines(out) + gab = lines.index( + "* :user:`Gabriel Changamire `" + " - :gh:`2809`" + ) + new = lines.index("* :user:`New User ` - :gh:`300`") + tob = lines.index("* :user:`Tobias Klauser ` - :gh:`2711`") + assert gab < new < tob + + def test_appends_gh_to_existing_contributor(self): + out, status = cb.apply_credit( + CREDITS, "tklauser", "Tobias Klauser", 300, "2026" + ) + assert status == "appended" + assert ( + "* :user:`Tobias Klauser ` - :gh:`2711`, :gh:`300`" + in self._lines(out) + ) + + def test_sorts_before_legacy_named_link(self): + out, status = cb.apply_credit( + CREDITS, "aardvark", "Aardvark Zero", 300, "2026" + ) + assert status == "added" + lines = self._lines(out) + new = lines.index("* :user:`Aardvark Zero ` - :gh:`300`") + amaan = lines.index("* `Amaan Qureshi`_ - :gh:`2770`") + assert new < amaan + + LEGACY = """\ +Code contributors +================= + +Code contributors by year +------------------------- + +2026 +~~~~ + +* `Amaan Qureshi`_ - :gh:`2770` +* :user:`Tobias Klauser ` - :gh:`2711` + +.. _`Amaan Qureshi`: https://github.com/amaanq +""" + + def test_repeat_legacy_contributor_appends_not_duplicates(self): + # Amaan is listed in the legacy `Name`_ style; his handle is in + # the link target. A second 2026 PR must append, not add a + # duplicate :user: line. + out, status = cb.apply_credit( + self.LEGACY, "amaanq", "Amaan Qureshi", 300, "2026" + ) + assert status == "appended" + lines = self._lines(out) + assert "* `Amaan Qureshi`_ - :gh:`2770`, :gh:`300`" in lines + assert not any(ln.startswith("* :user:`Amaan") for ln in lines) + + def test_giampaolo_is_skipped(self): + out, status = cb.apply_credit( + CREDITS, "giampaolo", "Giampaolo Rodola", 300, "2026" + ) + assert status == "skipped" + assert out == CREDITS + + def test_duplicate_gh_is_skipped(self): + out, status = cb.apply_credit( + CREDITS, + "gabrielchangamire-arch", + "Gabriel Changamire", + 2809, + "2026", + ) + assert status == "skipped" + assert out == CREDITS + + WRAPPED = """\ +Code contributors +================= + +Code contributors by year +------------------------- + +2026 +~~~~ + +* :user:`Long Name Person ` - :gh:`2001`, :gh:`2002`, :gh:`2003`, + :gh:`2004`, :gh:`2005` +* :user:`Short ` - :gh:`10` +""" + + def test_append_to_wrapped_entry_hits_last_line(self): + out, status = cb.apply_credit( + self.WRAPPED, "longname", "Long Name Person", 2006, "2026" + ) + assert status == "appended" + lines = self._lines(out) + # First physical line is untouched (no double comma). + first = lines.index( + "* :user:`Long Name Person ` - :gh:`2001`," + " :gh:`2002`, :gh:`2003`," + ) + assert ",," not in out + # The new ref lands on the continuation, not the first line. + assert ":gh:`2006`" in out + assert ":gh:`2006`" not in lines[first] + + def test_append_dup_check_spans_wrapped_entry(self): + # :gh:`2005` is on the continuation line; must still skip. + out, status = cb.apply_credit( + self.WRAPPED, "longname", "Long Name Person", 2005, "2026" + ) + assert status == "skipped" + assert out == self.WRAPPED + + def test_append_stays_within_79_cols(self): + out, _ = cb.apply_credit( + self.WRAPPED, "longname", "Long Name Person", 2006, "2026" + ) + assert all(len(ln) <= 79 for ln in self._lines(out)) + + def test_creates_new_year_block(self): + out, status = cb.apply_credit( + CREDITS, "newuser", "New User", 1, "2027" + ) + assert status == "added" + lines = self._lines(out) + assert "2027" in lines + y2027 = lines.index("2027") + y2026 = lines.index("2026") + # Newest year on top. + assert y2027 < y2026 + assert lines[y2027 + 1] == "~~~~" + + +class TestResortCredits: + def test_sorts_year_block(self): + text = """\ +Code contributors by year +------------------------- + +2026 +~~~~ + +* :user:`Tobias Klauser ` - :gh:`2711` +* `Amaan Qureshi`_ - :gh:`2770` +* :user:`Gabriel Changamire ` - :gh:`2809` +""" + out = cb.resort_credits_year(text, "2026") + entries = [x for x in out.splitlines() if x.startswith("* ")] + assert entries == [ + "* `Amaan Qureshi`_ - :gh:`2770`", + "* :user:`Gabriel Changamire ` - :gh:`2809`", + "* :user:`Tobias Klauser ` - :gh:`2711`", + ] + + def test_keeps_multiline_entry_together(self): + text = """\ +Code contributors by year +------------------------- + +2026 +~~~~ + +* :user:`Zoe Last ` - :gh:`10`, + :gh:`11` +* `Amaan Qureshi`_ - :gh:`2770` +""" + out = cb.resort_credits_year(text, "2026") + lines = out.splitlines() + zoe = lines.index("* :user:`Zoe Last ` - :gh:`10`,") + assert lines[zoe + 1] == " :gh:`11`" + assert lines.index("* `Amaan Qureshi`_ - :gh:`2770`") < zoe + + def test_leaves_other_years_untouched(self): + text = """\ +Code contributors by year +------------------------- + +2026 +~~~~ + +* :user:`Bea ` - :gh:`2` +* :user:`Ann ` - :gh:`1` + +2025 +~~~~ + +* :user:`Zed ` - :gh:`9` +* :user:`Amy ` - :gh:`8` +""" + out = cb.resort_credits_year(text, "2026") + lines = out.splitlines() + # 2026 got sorted, 2025 kept as-is. + assert lines.index("* :user:`Ann ` - :gh:`1`") < lines.index( + "* :user:`Bea ` - :gh:`2`" + ) + assert lines.index("* :user:`Zed ` - :gh:`9`") < lines.index( + "* :user:`Amy ` - :gh:`8`" + ) + + +class TestReferencedIssues: + def test_collects_refs_and_pr_number(self): + refs = cb.referenced_issues( + "Fix crash", "Closes #2809, see #100", 2810 + ) + assert refs == {2809, 100, 2810} + + def test_collects_url_and_gh_forms(self): + body = ( + "Fixes https://github.com/giampaolo/psutil/issues/2809\n" + "also GH-100 and giampaolo/psutil#42" + ) + refs = cb.referenced_issues("t", body, 2810) + assert {2809, 100, 42, 2810} <= refs + + +class TestApplyChangelogDecision: + ALLOWED = {200, 201, 300} + + def test_insert_valid(self): + decision = { + "action": "insert", + "section": "Bug fixes", + "entry_text": "- :gh:`300`: new bug.", + "amend_gh": None, + "skip_reason": None, + } + out, status, gh = cb.apply_changelog_decision( + CHANGELOG, decision, self.ALLOWED + ) + assert status == "inserted" + assert gh == 300 + assert "- :gh:`300`: new bug." in out + + def test_insert_rejects_issue_not_referenced(self): + decision = { + "action": "insert", + "section": "Bug fixes", + "entry_text": "- :gh:`777`: hallucinated.", + "amend_gh": None, + "skip_reason": None, + } + with pytest.raises(cb.ValidationError): + cb.apply_changelog_decision(CHANGELOG, decision, self.ALLOWED) + + def test_insert_rejects_malformed_entry(self): + decision = { + "action": "insert", + "section": "Bug fixes", + "entry_text": "just some text without a gh ref", + "amend_gh": None, + "skip_reason": None, + } + with pytest.raises(cb.ValidationError): + cb.apply_changelog_decision(CHANGELOG, decision, self.ALLOWED) + + def test_insert_rejects_when_issue_already_present(self): + # :gh:`200` already has an entry; insert should have been amend. + decision = { + "action": "insert", + "section": "Bug fixes", + "entry_text": "- :gh:`200`: duplicate.", + "amend_gh": None, + "skip_reason": None, + } + with pytest.raises(cb.ValidationError): + cb.apply_changelog_decision(CHANGELOG, decision, self.ALLOWED) + + def test_insert_rejects_entry_without_period(self): + decision = { + "action": "insert", + "section": "Bug fixes", + "entry_text": "- :gh:`300`: no period here", + "amend_gh": None, + "skip_reason": None, + } + with pytest.raises(cb.ValidationError): + cb.apply_changelog_decision(CHANGELOG, decision, self.ALLOWED) + + def test_amend_valid(self): + decision = { + "action": "amend", + "section": "Bug fixes", + "entry_text": "- :gh:`200`: bug one and also two.", + "amend_gh": 200, + "skip_reason": None, + } + out, status, _ = cb.apply_changelog_decision( + CHANGELOG, decision, self.ALLOWED + ) + assert status == "amended" + assert "- :gh:`200`: bug one and also two." in out + + def test_amend_allowed_when_issue_not_referenced_by_pr(self): + # An amend targets an entry that already exists in the block; + # the PR need not restate that issue number. + decision = { + "action": "amend", + "section": None, + "entry_text": "- :gh:`200`: bug one, extended.", + "amend_gh": 200, + "skip_reason": None, + } + out, status, _ = cb.apply_changelog_decision( + CHANGELOG, decision, {12345} + ) + assert status == "amended" + assert "- :gh:`200`: bug one, extended." in out + + def test_amend_rejects_when_gh_changed(self): + decision = { + "action": "amend", + "section": "Bug fixes", + "entry_text": "- :gh:`999`: wrong number.", + "amend_gh": 200, + "skip_reason": None, + } + with pytest.raises(cb.ValidationError): + cb.apply_changelog_decision(CHANGELOG, decision, self.ALLOWED) + + def test_skip_leaves_text_unchanged(self): + decision = { + "action": "skip", + "section": None, + "entry_text": None, + "amend_gh": None, + "skip_reason": "already covered.", + } + out, status, gh = cb.apply_changelog_decision( + CHANGELOG, decision, self.ALLOWED + ) + assert status == "skipped" + assert out == CHANGELOG + assert gh is None + + +class TestBuildComment: + def test_inserted(self): + decision = { + "action": "insert", + "section": "Bug fixes", + "entry_text": "- :gh:`300`: x.", + "amend_gh": None, + "skip_reason": None, + } + out = cb.build_comment("inserted", "added", decision, 300, "bob") + assert "entry added under **Bug fixes**" in out + assert "- :gh:`300`: x." in out + assert "credited @bob for :gh:`300`" in out + + def test_amended(self): + decision = { + "action": "amend", + "section": None, + "entry_text": "- :gh:`200`: y.", + "amend_gh": 200, + "skip_reason": None, + } + out = cb.build_comment("amended", "appended", decision, 200, "bob") + assert "entry for :gh:`200` amended" in out + assert "appended :gh:`200` to @bob" in out + + def test_skipped_states_reason_and_no_false_claim(self): + decision = { + "action": "skip", + "section": None, + "entry_text": None, + "amend_gh": None, + "skip_reason": "already covered.", + } + out = cb.build_comment("skipped", "skipped", decision, None, "bob") + assert "**not** modified (skipped): already covered." in out + assert "entry added" not in out + + +class TestRunDecision: + def _setup_files(self): + import tempfile + + d = pathlib.Path(tempfile.mkdtemp()) + clp = d / "changelog.rst" + crp = d / "credits.rst" + clp.write_text(CHANGELOG) + crp.write_text(CREDITS) + cb.CHANGELOG_FILE = str(clp) + cb.CREDITS_FILE = str(crp) + return clp, crp + + def test_amend_flow_writes_both_files(self): + # The incident scenario: an entry for the issue already exists, + # the PR amends it and the contributor is credited. + clp, _ = self._setup_files() + pr = { + "number": 2810, + "title": "Fix #200", + "body": "", + "author": "tklauser", + "author_name": "Tobias Klauser", + } + decision = { + "action": "amend", + "section": None, + "entry_text": "- :gh:`200`: bug one, now also two.", + "amend_gh": 200, + "skip_reason": None, + } + cl_status, cr_status, gh = cb.run_decision( + pr, decision, year="2026", get_labels=lambda _: [] + ) + assert cl_status == "amended" + assert gh == 200 + assert "- :gh:`200`: bug one, now also two." in clp.read_text() + # tklauser already credited in 2026 -> append. + assert cr_status == "appended" + + def test_skip_flow_touches_nothing(self): + clp, crp = self._setup_files() + before_cl = clp.read_text() + before_cr = crp.read_text() + pr = { + "number": 2810, + "title": "t", + "body": "", + "author": "tklauser", + "author_name": "Tobias Klauser", + } + decision = { + "action": "skip", + "section": None, + "entry_text": None, + "amend_gh": None, + "skip_reason": "already covered.", + } + cl_status, cr_status, _ = cb.run_decision( + pr, decision, year="2026", get_labels=lambda _: [] + ) + assert cl_status == "skipped" + assert cr_status == "skipped" + assert clp.read_text() == before_cl + assert crp.read_text() == before_cr + + def test_noop_amend_reports_skip(self): + # Amending an entry with byte-identical text changes nothing, so + # the bot must report a skip, not a false "amended". + clp, _ = self._setup_files() + before = clp.read_text() + pr = { + "number": 2810, + "title": "t", + "body": "", + "author": "tklauser", + "author_name": "Tobias Klauser", + } + decision = { + "action": "amend", + "section": None, + "entry_text": "- :gh:`200`: bug one.", + "amend_gh": 200, + "skip_reason": None, + } + cl_status, cr_status, gh = cb.run_decision( + pr, decision, year="2026", get_labels=lambda _: [] + ) + assert cl_status == "skipped" + assert cr_status == "skipped" + assert gh is None + assert clp.read_text() == before + + def test_insert_injects_badges_from_tracker_labels(self): + clp, _ = self._setup_files() + pr = { + "number": 2810, + "title": "Fix #300", + "body": "Fixes #300", + "author": "bob", + "author_name": "Bob", + } + decision = { + "action": "insert", + "section": "Bug fixes", + "entry_text": "- :gh:`300`, [Linux]: fix a double free.", + "amend_gh": None, + "skip_reason": None, + } + cl_status, _, _gh = cb.run_decision( + pr, + decision, + year="2026", + get_labels=lambda _: ["bug", "critical", "linux"], + ) + assert cl_status == "inserted" + text = clp.read_text() + entry = "- :gh:`300`, [Linux], :label:`critical`: fix a double free." + assert entry in text + # Labelled, so it leads the section. + lines = text.splitlines() + assert lines.index(entry) < lines.index("- :gh:`200`: bug one.") + # The comment shows the final entry, not the model's draft. + assert decision["entry_text"] == entry + + def test_model_written_badge_is_rejected(self): + self._setup_files() + pr = { + "number": 2810, + "title": "t", + "body": "Fixes #300", + "author": "bob", + "author_name": "Bob", + } + decision = { + "action": "insert", + "section": "Bug fixes", + "entry_text": "- :gh:`300`, :label:`critical`: sneaky.", + "amend_gh": None, + "skip_reason": None, + } + with pytest.raises(cb.ValidationError): + cb.run_decision(pr, decision, year="2026", get_labels=lambda _: []) + + def test_validation_failure_writes_nothing(self): + clp, _ = self._setup_files() + before_cl = clp.read_text() + pr = { + "number": 2810, + "title": "t", + "body": "", + "author": "bob", + "author_name": "Bob", + } + # :gh:`777` is not referenced by the PR -> rejected. + decision = { + "action": "insert", + "section": "Bug fixes", + "entry_text": "- :gh:`777`: hallucinated.", + "amend_gh": None, + "skip_reason": None, + } + with pytest.raises(cb.ValidationError): + cb.run_decision(pr, decision, year="2026", get_labels=lambda _: []) + assert clp.read_text() == before_cl + + +RELEASED_TOP = """\ +Changelog +========= + +8.0.1 (2026-02-01) +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`10`: released bug. +""" + + +class TestDecisionPostRelease: + def test_insert_allowed_even_if_issue_in_released_block(self): + # The released top block has :gh:`10`; a new PR for the same + # issue must still insert (into a fresh dev block), not be + # rejected as a duplicate. + decision = { + "action": "insert", + "section": "Bug fixes", + "entry_text": "- :gh:`10`: a follow-up fix.", + "amend_gh": None, + "skip_reason": None, + } + out, status, _ = cb.apply_changelog_decision( + RELEASED_TOP, decision, {10} + ) + assert status == "inserted" + assert "X.X.X (IN DEVELOPMENT)" in out + + def test_amend_rejected_when_top_block_released(self): + # Amending released history is never allowed. + decision = { + "action": "amend", + "section": None, + "entry_text": "- :gh:`10`: rewritten released bug.", + "amend_gh": 10, + "skip_reason": None, + } + with pytest.raises(cb.ValidationError): + cb.apply_changelog_decision(RELEASED_TOP, decision, {10}) + + +class TestReleasedBlock: + def test_creates_dev_block_when_top_is_released(self): + text = """\ +Changelog +========= + +8.0.1 (2026-02-01) +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`10`: released bug. +""" + out = cb.insert_entry(text, "Bug fixes", "- :gh:`300`: new bug.") + lines = out.splitlines() + dev = lines.index("X.X.X (IN DEVELOPMENT)") + released = lines.index("8.0.1 (2026-02-01)") + new = lines.index("- :gh:`300`: new bug.") + # New dev block sits above the released one, entry inside it. + assert dev < new < released + assert lines[dev + 1] == "^" * len("X.X.X (IN DEVELOPMENT)") + + +class TestGhRequest: + class _Resp: + def __init__(self, body): + self.body = body + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def read(self): + return self.body + + def _call(self, result): + def fake_urlopen(req, timeout=None): + if isinstance(result, Exception): + raise result + return self._Resp(result) + + orig_open = cb.urllib.request.urlopen + cb.urllib.request.urlopen = fake_urlopen + cb.TOKEN = "x" + try: + return cb.gh_request("/x") + finally: + cb.urllib.request.urlopen = orig_open + + def _http_error(self, code): + return cb.urllib.error.HTTPError( + "http://x", code, "err", {}, io.BytesIO(b"boom") + ) + + def test_returns_body(self): + assert self._call(b"ok") == b"ok" + + def test_error_includes_github_body(self): + # A bare "HTTP Error 404" hides why GitHub refused. + err = self._http_error(404) + try: + with pytest.raises(SystemExit, match="boom"): + self._call(err) + finally: + err.close() diff --git a/.github/workflows/tests/test_triage_labels.py b/.github/workflows/tests/test_triage_labels.py new file mode 100644 index 0000000000..ba8fb4dec2 --- /dev/null +++ b/.github/workflows/tests/test_triage_labels.py @@ -0,0 +1,90 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for the triage bot (.github/workflows/triage_labels.py). + +These cover the confidence gating, which is what keeps a shaky answer +from deleting a label someone applied by hand. No network or API is +used. +""" + +import importlib.util +import pathlib + +BOT_PATH = pathlib.Path(__file__).parent.parent / "triage_labels.py" + + +def import_module_by_path(path): + spec = importlib.util.spec_from_file_location(path.stem, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +tl = import_module_by_path(BOT_PATH) + + +def decide(**kw): + """A decision, confident and empty unless told otherwise. + + Pass e.g. platform=["linux"], platform_confidence="low". + """ + out = { + "type": "bug", + "platform": [], + "component": [], + "severity": [], + } + out.update({f"{axis}_confidence": "high" for axis in tl.AXES}) + out.update(kw) + return out + + +class TestFreshLabels: + def test_low_confidence_axis_contributes_nothing(self): + decision = decide( + platform=["linux"], + platform_confidence="low", + component=["tests"], + ) + assert tl.fresh_labels(decision) == {"bug", "tests"} + + def test_low_confidence_type_contributes_nothing(self): + decision = decide(type="bug", type_confidence="low") + assert tl.fresh_labels(decision) == set() + + +class TestStaleLabels: + def test_removes_only_when_confident(self): + item = {"labels": ["bug", "windows"]} + decision = decide(platform=["linux"], platform_confidence="medium") + assert tl.stale_labels(item, decision) == set() + + decision = decide(platform=["linux"], platform_confidence="high") + assert tl.stale_labels(item, decision) == {"windows"} + + def test_severity_is_never_removed(self): + # severity is add-only: the text can suggest critical but it + # can never prove the absence of one. + item = {"labels": ["bug", "critical", "memleak"]} + decision = decide(severity=[], severity_confidence="high") + assert tl.stale_labels(item, decision) == set() + + def test_confident_null_type_clears_bug(self): + item = {"labels": ["bug", "linux"]} + decision = decide(type=None, platform=["linux"]) + assert tl.stale_labels(item, decision) == {"bug"} + + +class TestDropGeneralPlatforms: + def test_named_os_wins_over_the_general_one(self): + decision = decide(platform=["unix", "linux"]) + assert tl.fresh_labels(decision) == {"bug", "linux"} + + decision = decide(platform=["bsd", "freebsd"]) + assert tl.fresh_labels(decision) == {"bug", "freebsd"} + + def test_general_one_stays_when_nothing_names_an_os(self): + decision = decide(platform=["unix", "pypy"]) + assert tl.fresh_labels(decision) == {"bug", "unix", "pypy"} diff --git a/.github/workflows/triage.py b/.github/workflows/triage.py new file mode 100755 index 0000000000..a0c5afeabc --- /dev/null +++ b/.github/workflows/triage.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Bot triggered by Github Actions every time a new issue or PR is +created. Replies to common mistakes. Labelling is +.github/workflows/triage_labels.py's job. +""" + +import functools +import json +import os +import pathlib +from pprint import pprint as pp + +from github import Github + +ROOT_DIR = pathlib.Path(__file__).resolve().parent.parent.parent +MAINTAINERS = {"giampaolo"} + +# --- replies + +REPLY_MAINTAINER_OWNED_FILES = """\ +âš ï¸ Please **remove** your changes to `docs/changelog.rst` and / or \ +`docs/credits.rst`. âš ï¸ +These two files are maintained by the project, and a \ +maintainer edits them before the PR is merged. \ +Editing them in a PR tends to cause merge conflicts. \ +This is an auto-generated response. +""" + + +# --- github API utils + + +def is_pr(issue): + return issue.pull_request is not None + + +def get_repo(): + repo = os.environ['GITHUB_REPOSITORY'] + token = os.environ['GITHUB_TOKEN'] + return Github(token).get_repo(repo) + + +# --- event utils + + +@functools.lru_cache +def _get_event_data(): + with open(os.environ["GITHUB_EVENT_PATH"]) as f: + ret = json.load(f) + pp(ret) + return ret + + +def is_event_new_pr(): + data = _get_event_data() + try: + return data['action'] == 'opened' and 'pull_request' in data + except KeyError: + return False + + +def get_issue(): + data = _get_event_data() + try: + num = data['issue']['number'] + except KeyError: + num = data['pull_request']['number'] + return get_repo().get_issue(number=num) + + +# --- actions + + +def log(msg): + if '\n' in msg or "\r\n" in msg: + print(f">>>\n{msg}\n<<<", flush=True) + else: + print(f">>> {msg} <<<", flush=True) + + +def on_new_pr(issue): + if issue.user.login in MAINTAINERS: + return + pr = get_repo().get_pull(issue.number) + files = [x.filename for x in pr.get_files()] + + # changelog.rst / credits.rst are maintainer-owned; ask to drop them. + owned = ("docs/changelog.rst", "docs/credits.rst") + if any(f in files for f in owned): + log("PR edits maintainer-owned changelog/credits files") + issue.create_comment(REPLY_MAINTAINER_OWNED_FILES) + + +def main(): + issue = get_issue() + stype = "PR" if is_pr(issue) else "issue" + log(f"running issue bot for {stype} {issue!r}") + + if is_event_new_pr(): + log(f"created new PR {issue}") + on_new_pr(issue) + else: + log("unhandled event") + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/triage.yml b/.github/workflows/triage.yml new file mode 100644 index 0000000000..a9502fd8d5 --- /dev/null +++ b/.github/workflows/triage.yml @@ -0,0 +1,42 @@ +# Fired by Github Actions every time an issue or PR is created. + +name: triage +on: + issues: + types: [opened] + pull_request_target: + types: [opened] +permissions: + issues: write + pull-requests: write +jobs: + triage: + runs-on: ubuntu-latest + steps: + # No `ref:` on purpose. pull_request_target runs with our + # secrets, so checking out the PR's head would hand them over. + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.x" + + - name: Install deps + run: python3 -m pip install PyGithub anthropic + + - name: Reply + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PYTHONUNBUFFERED=1 PYTHONWARNINGS=always python3 .github/workflows/triage.py + + - name: Label + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + NUMBER: ${{ github.event.issue.number || github.event.pull_request.number }} + run: | + PYTHONUNBUFFERED=1 python3 .github/workflows/triage_labels.py "$NUMBER" --apply diff --git a/.github/workflows/triage_labels.py b/.github/workflows/triage_labels.py new file mode 100755 index 0000000000..03d42e995f --- /dev/null +++ b/.github/workflows/triage_labels.py @@ -0,0 +1,868 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Setup the right labels for new GitHub issues and PRs by asking Claude. + +Usage: + python3 .github/workflows/triage_labels.py 2635 + python3 .github/workflows/triage_labels.py 2635 1783 2029 + python3 .github/workflows/triage_labels.py 2635 --apply +""" + +import argparse +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request + +REPO = "giampaolo/psutil" +HTTP_TIMEOUT = 30 +MAX_BODY_CHARS = 6000 +MAX_FILES = 100 +MAX_TOKENS = 2048 + +# Set by parse_cli(). +TOKEN = "" +MODEL = "" +NUMBERS = [] +APPLY = False + +PROMPT = """\ +You are triaging a psutil issue or pull request. psutil is a Python +library that reads process and system information, with a Python layer +per platform (_pslinux.py, _pswindows.py, ...) backed by C extensions. + +Type is bug or null. Platform, component and severity are lists and +may name more than one, though most items need a platform and little +else. On the three lists, leave it empty rather than reaching for a +label that only half fits: a wrong label is worse than no label. + +TYPE + +- bug: something is broken, wrong, or crashes. + +That is the only type. There is no label for the rest: a new feature, +a speedup, a refactor or a question simply isn't a bug, so answer +null, and let the component list say which kind of not-a-bug it is. A +confident null is also what clears a bug label that shouldn't be +there. + +The template's "Bug fix: yes/no" line is a hint, not the answer. Read +what the change does. Adding support for something that never worked +is not a bug even when the author ticked yes. + +PLATFORM + +Fill this only when the item is specific to where psutil runs: an OS, +a container, a different cPython implementation (PYPY). A bug that would +happen anywhere is an empty list, even when the reporter happens to be +on Linux. + +A "[Linux]" tag in the title or a filled-in "* OS: ..." line is the +reporter saying it outright, so take them at their word. When they +name two or three, list all of them. These mix freely, so a container +bug on Linux is ["linux", "vm"]. + +On a PR, the changed files outrank that line. People fill the template +in loosely and it is often stale or plain wrong: a PR whose only file +is .github/workflows/tests.yml has no platform, whatever its "OS:" +line claims. Believe the diff. + +Going wide is the opposite of specific, so leave it empty. A sweep +across every arch/ directory, a refactor of shared code, anything that +lands everywhere: no platform at all. Four or more is nearly always +this mistake. Don't read a PR's changed files as a list of platforms +to claim. + +That rule is about the diff, not the title. Names written in the title +always count, tagged or not: "[Windows/Linux/Mac] ..." and "publish +macos and linux wheels" each name platforms out loud, so list them. +An environment counts as the OS it runs on, so cygwin and msys are +windows. + +One path does settle it. psutil/arch/ holds a directory per platform, +and a diff that stays inside one of them says which: arch/windows/ is +windows, arch/osx/ is macos, arch/solaris/ is sunos, arch/bsd/ is bsd, +arch/posix/ is unix. Only arch/all/ is everywhere. Land in two of them +and you're back to the sweep above. + +An OS named in passing is not the subject either. "Known cases are +AccessDenied on Windows and a null ctime on NetBSD" is a cross-platform +bug illustrated with examples, so the list stays empty. "OS: all" means +empty no matter which names follow it. Ask what the fix changes, not +where the symptom was noticed. + + +- linux, windows, macos, freebsd, openbsd, netbsd, sunos, aix: the + item is about that OS. + +- bsd: the item is about the BSDs as a family. "on all 3 BSDs", a + "[BSD]" tag, a fix in the shared psutil/arch/bsd/ code. Listing the + three by name changes nothing: "all 3 BSDs (FreeBSD, OpenBSD, + NetBSD)" is still one bsd label, not three. Reach for the specific + ones only when the item is about some of them but not all. + +- unix: very rare. Shared POSIX code across several unices where no + single OS fits and the item names none. Something POSIX has and + Windows doesn't counts even with nothing named: zombie processes, + signals, uid and gid, fork, terminals. So does a diff confined to + psutil/arch/posix/ or _psutil_posix.c. It stands alone: the moment + you can name one OS, list that instead. + +- vm: any container or virtual OS, Docker included. Only when it's + material, not merely where the reporter happened to run. + +- pypy: the item is about running under PyPy, not CPython. + +COMPONENT + +Usually empty. Roughly half of all items are just a platform bug with +no component at all. Two is common enough: a cibuildwheel change in a +workflow file is ["packaging", "ci"]. Three is rarer but real, and a +CI change to the wheel build that is also a speedup earns all three. +Four is almost certainly wrong. + +The rule for all of them: the item has to be *specific* to the +component, not merely touch it. A new feature updates the docs, adds +tests and maybe a script, and it is still just the feature. Only reach +for a component label when it is what the item is for. These get +over-applied, so the labels already in the repo are a poor guide. When +in doubt, leave the list empty. + + +- doc: prose under docs/, the README, docstrings, the doc build or + theme. A docstring-only fix counts even though it lives in a .py + file. A feature or bugfix that updates the docs on the way past + does not: that one is the feature or the bug. + +- tests: the test suite and nothing else. A flaky test, a slow test, a + test asserting the wrong thing, a skip, a test helper. For a PR the + changed files settle it: touching library code (psutil/*.py, + psutil/arch/, the C extensions) means the PR is about that code, so + no; tests plus boilerplate like HISTORY.rst or the Makefile is fine. + A reported test failure that turns out to be a real bug is that bug, + and the PR fixing it gets the bug's labels, never this one. + +- ci: psutil's own automation. Anything under .github/workflows/, plus + cirrus, appveyor and travis. A "CI:" prefix in the title says it + outright, so take it. The runners and the test matrix, but + equally the bots and release jobs that never run a test: a changed + workflow file is nearly always this. Also a job failing for reasons + unrelated to the code under test. + +- scripts: psutil's own scripts/ directory, including the examples. + Not the reporter's script. People often paste one to show a bug; + that bug is about whatever it exercises. + +- internals: psutil's own machinery, with no effect on the public + API. Debug output, internal helpers and refactorings, the Makefile, + linters and formatters. ci, tests and scripts are its more specific + siblings, so reach for internals only when none of them fits. + +- packaging: what psutil ships and how it's built. Wheels, the sdist, + the build backend, what gets installed. The release matrix, + manylinux, a wheel missing from PyPI. cibuildwheel settles it on + its own: an item that touches it is about packaging, even when the + change is to the workflow around it, in which case it is ci as + well. A compile error on the reporter's own machine is the + build-fail severity instead. + +- new-api: the public API grows. A brand new function or method, but + equally a new argument on one that already exists, a new field in a + namedtuple it returns, a new value it can now give back. Anything + that hands callers something they couldn't reach before. Making an + existing call work on one more platform is not this. + +- api-change: an existing public API changed, was deprecated or was + removed. A rename, a changed field, a different return type, a new + deprecation warning. The counterpart of new-api, which is for + growth. When working code has to change, add compatibility too. + +- performance: speed or resource usage is the point. Slow is + performance, wrong is a bug. psutil's own build and CI count too: + making the suite, the wheel build or a workflow faster is + performance, on top of ci or packaging. A timing table, or a + benchmark showing timings before and after, is the giveaway. So is + releasing and reacquiring the GIL around a blocking syscall, + numbers or no numbers: the whole point is letting other threads + run. + +- compatibility: psutil's support matrix moves, or what callers can + rely on does. Dropping an old Python or OS version, or restoring + one psutil had lost. Dropping a wheel target or an interpreter + build, removing a dependency that moves the floor psutil builds + on, removing or renaming a public API, dropping a field from a + namedtuple. The test is whether a working install or working code + has to change. Correcting a value that was simply wrong is not + this, it's the bug fix, and a one-off build error on a platform + psutil already supports is a plain bug, not a change of support. + +- dropped-support: a platform, OS version or Python version is no + longer supported. Nearly always compatibility as well, since + installs that worked have to change. + +- new-platform: support for an operating system psutil does not target + yet. + +SEVERITY + +Ways a bug can be worse than a wrong answer. Usually none applies. +More than one at once is rare, but allowed. These become colored +badges in the changelog, so a wrong one is very visible. + +- critical: the process doesn't survive, or its memory is no longer + trustworthy. A segfault, a use-after-free, a double free, a buffer + overflow, an abort. A deadlock or a hang counts too: the process is + still there but it's never coming back. So does blowing up at + import time: whatever the exception, ``import psutil`` took the + program down before it started. A DLL or extension that won't load + is build-fail instead, since that build never worked. + +- build-fail: psutil doesn't compile or link. A missing header, an + undeclared constant, an undefined symbol, a compiler that chokes on + the source. The reporter's own toolchain counts: no Python.h, no C + compiler installed, the wrong MSVC. So does an extension that built + but won't load for an undefined symbol. A test that fails, a + compile warning and a wheel missing from PyPI are not this. It + never pairs with critical: nothing ever ran, so no runtime failure + applies. + +- memleak: memory is leaked. Growth without bound, but also a single + allocation, handle or refcount never released, error paths + included. If the text says leak and points at what leaks, that's + this. + +- badexc: psutil raises something it isn't allowed to. The public API + may raise NoSuchProcess, AccessDenied, ZombieProcess and + TimeoutExpired, and nothing else. Anything else getting out is this: + a RuntimeError, a SystemError, an OSError, a KeyError, an IndexError, + a UnicodeDecodeError. FileNotFoundError and PermissionError count as + well, being exactly what psutil should have turned into NoSuchProcess + and AccessDenied. + + This is about the type, never the timing. One of those four raised + when it shouldn't have been, a false NoSuchProcess on a process that + is still alive say, is a wrong answer: a plain bug, not badexc. + + Near misses, none of them badexc: an AssertionError is a test + failing. An ImportError or a DLL that won't load is a build that + didn't work. An AttributeError on a name that's gone is a caller on + an old API. NotImplementedError is how psutil says the platform + can't answer. A warning is not an exception. ValueError and + TypeError on a bad argument are the API working, though one escaping + a /proc or registry parse does count. + +It has to be psutil doing it: people paste whole tracebacks from +whatever program hit the problem, so find psutil in the failing frame +first. A wrong value, a slow call and a leak are plain bugs however +annoying. So is a build that won't compile, which never got as far as +running. An umbrella issue cataloguing ten crashes is about the audit, +not any one crash, but a PR that fixes several things carries all of +them: "[SunOS] various fixes" can end up with both labels. + +CONFIDENCE + +Give type, platform, component and severity a confidence. Use low when +the text is too thin to tell, so the choice can be discarded later. For +type, platform and component an empty answer with high confidence means +you are sure nothing applies, and is what lets a wrong label already on +the ticket be cleared. Severity is only ever added, never taken away, +so an empty one says nothing about what the ticket already carries. + +EXAMPLES + +Title: "Process.memory_info() returns 0 for all processes on Windows 11" +type=bug, platform=["windows"], component=[]. A plain platform bug, +which is the most common shape. No component label applies. + +Title: "add Process.num_threads() to the AIX implementation" +type=null, platform=["aix"], component=["new-api"]. + +Title: "test_disk_partitions fails on the macOS runner since the image +bump" +type=bug, platform=["macos"], component=["ci"]. The suite is fine; the +runner image changed. Not tests. + +Title: "test_cpu_percent asserts the wrong bound" +type=bug, platform=[], component=["tests"]. The test code is wrong, and +it is wrong everywhere. + +Title: "[SunOS] test_unix fails: invalid kind argument 'unix'" +type=bug, platform=["sunos"], component=[]. A test is how this +surfaced, but net_connections() really is missing a kind on SunOS. +Fix the code and the test goes green, so the bug is the item. + +Title: "cpu_times() is 3x slower than it needs to be" +type=null, component=["performance"]. Slow, not wrong. + +Title: "[OpenBSD, NetBSD] build failed" +type=bug, platform=["openbsd", "netbsd"], severity=["build-fail"]. +Both named, so both go in. Not bsd. + +Title: "IOError on import when /proc/stat is inaccessible" +type=bug, platform=["linux"], severity=["critical"]. The build is +fine; ``import psutil`` itself blows up, taking the program with it. + +Title: "macOS: fix SystemError in Process.cmdline() and environ()" +type=bug, platform=["macos"], severity=["badexc"]. SystemError isn't +one of the four psutil is allowed to raise, so it counts however +small the fix turns out to be. + +Title: "False NoSuchProcess('PID has been reused') on a process that is +still alive" +type=bug, platform=[], severity=[]. NoSuchProcess *is* one of the four. +Raising it at the wrong moment is a wrong answer, not badexc. + +Title: "[Windows] win_service_iter() can segfault on enumeration +failure" +type=bug, platform=["windows"], severity=["critical"]. The process +dies. Nothing was raised, so no badexc. + +Title: "[Windows] net_if_stats() reports the wrong link speed" +type=bug, platform=["windows"], severity=[]. A wrong number is a +plain bug. Nothing got out and nothing died. + +Title: "Fix refcount leaks on parse failure (Linux disk_partitions, +SunOS proc)" +type=bug, platform=["linux", "sunos"], severity=["memleak"]. Both +named, and a leak down an error path is still a leak. + +Title: "Rename Process.connections() to Process.net_connections()" +type=null, component=["api-change", "compatibility"]. An existing API +moved, and code that wants to stay warning-free has to follow. + +Title: "Drop Python 3.6 and 3.7" +type=null, component=["dropped-support", "compatibility"]. Installs +that worked have to change. No platform: this isn't about where +psutil runs. + +Title: "Upgrade cibuildwheel to 4.1.1, drop cp313t wheels" +type=null, component=["packaging", "ci", "compatibility"]. +cibuildwheel means packaging, it lands in a workflow so ci, and +dropping a build target narrows what we ship. Three is unusual and +here it's right. + +Title: "docs: add explanatory comments to the README examples" +type=null, component=["doc"]. Prose and nothing else, so doc is what +the item is for rather than something it touched on the way past. + +Answer with the submit tool.""" + +# Kept out of PROMPT so the cached prefix is byte-identical between +# tickets. Anything ticket-specific has to live after the breakpoint. +TICKET = """\ +Kind: {kind} +Title: {title} + +Body: +{body} +{files}""" + + +# --- the label taxonomy, as axes +# +# The axis names come from the label descriptions on GitHub. + +TYPE_LABELS = ["bug"] +PLATFORM_LABELS = [ + "linux", "windows", "macos", "freebsd", "openbsd", "netbsd", "bsd", + "sunos", "aix", "unix", "vm", "pypy", +] # fmt: skip +SEVERITY_LABELS = ["critical", "badexc", "build-fail", "memleak"] +COMPONENT_LABELS = [ + "doc", "tests", "ci", "scripts", "internals", "packaging", + "new-api", "api-change", "performance", "compatibility", + "dropped-support", "new-platform", +] # fmt: skip + +# The model never sees these, and they're stripped before comparing. +IGNORED_LABELS = { + "imported", + "need-more-info", + "dependencies", + "github_actions", +} + +AXES = ("type", "platform", "component", "severity") +LIST_AXES = ("platform", "component", "severity") +AXIS_LABELS = { + "type": TYPE_LABELS, + "platform": PLATFORM_LABELS, + "component": COMPONENT_LABELS, + "severity": SEVERITY_LABELS, +} +# severity is missing on purpose. The text can suggest it but never +# rule it out, so we add those and never take them away. +REMOVABLE_AXES = ("type", "platform", "component") + + +def enum_list(labels, description): + return { + "type": "array", + "items": {"type": "string", "enum": labels}, + "description": description, + } + + +GENERAL_PLATFORMS = { + "bsd": {"freebsd", "openbsd", "netbsd"}, + "unix": { + "aix", + "bsd", + "freebsd", + "linux", + "macos", + "netbsd", + "openbsd", + "sunos", + }, +} + + +def drop_general_platforms(labels): + """Drop unix / bsd when the same answer also names an OS.""" + out = set(labels) + for general, specific in GENERAL_PLATFORMS.items(): + if out & specific: + out.discard(general) + return out + + +def axis_values(decision, axis): + """What a decision puts on one axis, always as a set.""" + value = decision[axis] + if axis in LIST_AXES: + return set(value) + return {value} if value else set() + + +CONFIDENCE = {"type": "string", "enum": ["high", "medium", "low"]} + +DECISION_PROPS = { + "type": { + "anyOf": [ + {"type": "string", "enum": ["bug"]}, + {"type": "null"}, + ], + "description": "bug when something is broken, else null.", + }, + "type_confidence": CONFIDENCE, + "platform": enum_list( + PLATFORM_LABELS, + "Every OS, container or interpreter the item is specific to." + " Often empty.", + ), + "platform_confidence": CONFIDENCE, + "component": enum_list( + COMPONENT_LABELS, + "What the item is specifically about. Usually empty, sometimes two.", + ), + "component_confidence": CONFIDENCE, + "severity": enum_list( + SEVERITY_LABELS, + "critical when the process dies or memory is corrupt," + " build-fail when psutil won't compile, memleak when memory is" + " leaked, badexc when psutil raises something it shouldn't." + " Usually empty.", + ), + "severity_confidence": CONFIDENCE, +} + +SUBMIT_TOOL = { + "name": "submit", + "description": "Submit the label decision for this issue or PR.", + # Without this the schema is advisory: seen returning a single + # out-of-enum field, and nesting the payload under "parameter name". + "strict": True, + "input_schema": { + "type": "object", + "additionalProperties": False, + "properties": DECISION_PROPS, + "required": list(DECISION_PROPS), + }, +} + +# --- github + + +def gh_request(path, post=None, method=None): + req = urllib.request.Request( + f"https://api.github.com{path}", + data=json.dumps(post).encode() if post else None, + method=method, + headers={ + "Authorization": f"Bearer {TOKEN}", + "Accept": "application/vnd.github+json", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as resp: + return json.loads(resp.read()) + except urllib.error.HTTPError as err: + body = err.read().decode("utf-8", errors="replace") + sys.exit(f"GitHub API {err.code} for {path}: {body}") + + +def fetch_item(number): + """One issue or PR, in the shape classify() wants.""" + raw = gh_request(f"/repos/{REPO}/issues/{number}") + item = { + "number": raw["number"], + "title": raw["title"], + "body": raw.get("body") or "", + "is_pr": "pull_request" in raw, + "labels": sorted(x["name"] for x in raw["labels"]), + "files": [], + "by_bot": (raw.get("user") or {}).get("type") == "Bot", + } + if item["is_pr"]: + files = gh_request(f"/repos/{REPO}/pulls/{number}/files") + item["files"] = [f["filename"] for f in files][:MAX_FILES] + return item + + +# "Fixes #123", "closes gh-123", or the full issue URL. +CLOSES = re.compile( + r"\b(?:fix(?:e[sd])?|close[sd]?|resolve[sd]?)\b[\s:]*" + r"(?:https?://github\.com/[\w.-]+/[\w.-]+/issues/|gh-|#)(\d+)", + re.IGNORECASE, +) + + +def closed_issues(item): + if not item["is_pr"]: + return [] + seen = [] + for match in CLOSES.finditer(item["body"][:MAX_BODY_CHARS]): + number = int(match.group(1)) + if number != item["number"] and number not in seen: + seen.append(number) + return seen + + +def inherit_from_closed(labels, issue_labels): + """Take critical, build-fail and memleak from the issue a PR + closes. + + The issue quotes the traceback, the PR just says "handle EFAULT", + so the same defect reads as critical on one and not the other. + + Sharing a platform is what says the PR really is the fix. "chore: + test with Python 3.12" closes a Windows bug without being its fix + and inherits nothing. + """ + out = set(labels) + ours = out & set(PLATFORM_LABELS) + theirs = set(issue_labels) & set(PLATFORM_LABELS) + if (ours & theirs) or not (ours or theirs): + out |= {"critical", "build-fail", "memleak"} & set(issue_labels) + return out + + +def add_labels(number, labels): + """Add labels to a ticket. This endpoint never removes any.""" + gh_request( + f"/repos/{REPO}/issues/{number}/labels", {"labels": sorted(labels)} + ) + + +def remove_label(number, label): + path = urllib.parse.quote(label) + gh_request(f"/repos/{REPO}/issues/{number}/labels/{path}", method="DELETE") + + +def fresh_labels(decision): + """The labels a decision is willing to stand behind. + + Low means the model is guessing, so that axis contributes nothing. + Medium still counts: a third of the corpus lands there and it's + right most of the time. + """ + out = set() + for axis in AXES: + if decision[f"{axis}_confidence"] != "low": + out |= axis_values(decision, axis) + return drop_general_platforms(out) + + +def stale_labels(item, decision, from_bot=()): + """Labels the model just contradicted on the same axis. + + Only where it was sure, since medium or low means "I can't tell" + and that's no reason to delete what a person put there. A confident + empty answer does count, and is the only way a wrong label ever + gets cleared. + + from_bot is what the old regex bot applied; that comes off on + medium too. + """ + keep = model_labels(decision) + out = set() + for axis in REMOVABLE_AXES: + conf = decision[f"{axis}_confidence"] + if conf == "high": + out |= {x for x in item["labels"] if x in AXIS_LABELS[axis]} - keep + elif conf == "medium" and axis == "component": + # The bot read components off the template's "Type:" line, + # which reporters fill in by ticking everything. Its + # platforms came from "[Linux]" title tags and are usually + # right, so those stay protected. + botted = {x for x in item["labels"] if x in AXIS_LABELS[axis]} + out |= (botted & set(from_bot)) - keep + return out + + +# --- the model + + +class BadDecision(Exception): + """The model's tool call doesn't match the schema.""" + + +def allowed_values(prop): + """The values a schema property accepts, None if unconstrained.""" + if "enum" in prop: + return prop["enum"] + if prop.get("type") == "array": + return prop["items"]["enum"] + for branch in prop.get("anyOf", []): + if "enum" in branch: + return [*branch["enum"], None] + return None + + +def build_prompt(title, body, files): + listing = "" + if files: + names = "\n".join(f"- {f}" for f in files) + listing = f"\nChanged files:\n{names}\n" + return TICKET.format( + # An issue has no changed files, a PR always has at least one. + kind="pull request" if files else "issue", + title=title, + body=body[:MAX_BODY_CHARS] or "(empty)", + files=listing, + ) + + +def check_decision(decision): + """Fail loud on a decision that doesn't match the schema. + + strict=True should make this unreachable, but a malformed decision + reads downstream as "no labels" and quietly poisons the result, + which has already happened once. + """ + unknown = set(decision) - set(DECISION_PROPS) + missing = set(DECISION_PROPS) - set(decision) + if unknown or missing: + raise BadDecision( + f"bad keys (unknown={sorted(unknown)}," + f" missing={sorted(missing)}): {decision}" + ) + for name, value in decision.items(): + allowed = allowed_values(DECISION_PROPS[name]) + if allowed is None: + continue + if not isinstance(value, list): + value = [value] + elif len(set(value)) != len(value): + raise BadDecision(f"{name}={value!r} has duplicates") + for one in value: + if one not in allowed: + raise BadDecision(f"{name}={one!r} not in {allowed}") + + +def thinking_kwargs(): + if MODEL.startswith( + ("claude-opus-5", "claude-sonnet-5", "claude-fable-5") + ): + return { + "thinking": {"type": "adaptive"}, + "output_config": {"effort": "low"}, + } + return {} + + +def classify(client, title, body, files): + """Ask Claude which labels apply. + + Returns (decision, usage). Pass files=None for an issue. Raises + BadDecision when the tool call doesn't validate, so a bad answer + can't pass for an empty one. + """ + message = client.messages.create( + model=MODEL, + max_tokens=MAX_TOKENS, + **thinking_kwargs(), + tools=[SUBMIT_TOOL], + tool_choice={"type": "tool", "name": "submit"}, + # No caching. The 5 minute TTL never survives to the next + # issue, so it only ever pays for the write. + system=PROMPT, + messages=[ + {"role": "user", "content": build_prompt(title, body, files)} + ], + ) + if message.stop_reason == "max_tokens": + raise BadDecision("response truncated (raise MAX_TOKENS)") + block = next((b for b in message.content if b.type == "tool_use"), None) + if block is None: + raise BadDecision(f"no tool call (stop_reason={message.stop_reason})") + check_decision(block.input) + return block.input, message.usage + + +def model_labels(decision): + """Flatten a decision into the label set it implies.""" + out = set() + for axis in AXES: + out |= axis_values(decision, axis) + return out + + +# --- cli + + +def make_client(): + import anthropic + + key = os.environ.get("ANTHROPIC_API_KEY", "").strip() + if not key: + path = os.path.expanduser("~/.anthropic.api.key") + if not os.path.exists(path): + sys.exit(f"no ANTHROPIC_API_KEY and no {path}") + with open(path) as f: + key = f.read().strip() + return anthropic.Anthropic(api_key=key) + + +def fmt(labels): + return ", ".join(sorted(labels)) if labels else "-" + + +def report(item, decision): + kind = "PR" if item["is_pr"] else "issue" + print(f"#{item['number']} ({kind}) {item['title']}") + for axis in AXES: + conf = decision.get(f"{axis}_confidence") + suffix = f" ({conf})" if conf else "" + print(f" {axis:12s} {fmt(axis_values(decision, axis))}{suffix}") + print(f" already has: {fmt(set(item['labels']) - IGNORED_LABELS)}") + + +def parse_cli(): + global TOKEN, MODEL, NUMBERS, APPLY + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("numbers", nargs="+", type=int, help="issue or PR numbers") + p.add_argument( + "--token", + default="~/.github.api.key", + help="file holding a GitHub token. GITHUB_TOKEN wins.", + ) + p.add_argument("--model", default="claude-sonnet-5") + p.add_argument( + "--apply", + action="store_true", + help="add the labels on GitHub; without this nothing is written", + ) + args = p.parse_args() + TOKEN = os.environ.get("GITHUB_TOKEN", "").strip() + if not TOKEN: + with open(os.path.expanduser(args.token)) as f: + TOKEN = f.read().strip() + MODEL = args.model + NUMBERS = args.numbers + APPLY = args.apply + + +def show_tokens(prefix, usage): + # input_tokens excludes the cache write, which is most of it. + print( + f" {prefix:12s} {usage.input_tokens} in," + f" {usage.cache_creation_input_tokens} written," + f" {usage.cache_read_input_tokens} cached," + f" {usage.output_tokens} out" + ) + + +def handle(item, decision, usage, totals, index): + """Print one decision and, with --apply, act on it.""" + if index: + print() + report(item, decision) + if usage is not None: + for field in totals: + totals[field] += getattr(usage, field) + show_tokens("tokens:", usage) + judged = fresh_labels(decision) + for number in closed_issues(item): + try: + linked = fetch_item(number)["labels"] + judged = inherit_from_closed(judged, linked) + except SystemExit: + # The issue may be gone, or in another repo. Not a reason + # to give up on labelling the PR. + print(f" (couldn't read #{number}, ignoring the link)") + add = judged - set(item["labels"]) + drop = stale_labels(item, decision) + print(f" to add: {fmt(add)}") + print(f" to drop: {fmt(drop)}") + if not (add or drop): + return + if not APPLY: + print(" (--apply to do it)") + return + if add: + add_labels(item["number"], add) + for label in sorted(drop): + remove_label(item["number"], label) + print(" applied") + + +def run(totals): + client = make_client() + for index, number in enumerate(NUMBERS): + item = fetch_item(number) + if item["by_bot"]: + print(f"#{number}: opened by a bot, skipping") + continue + try: + decision, usage = classify( + client, item["title"], item["body"], item["files"] + ) + except BadDecision as err: + sys.exit(f"#{number}: {err}") + handle(item, decision, usage, totals, index) + + +def main(): + parse_cli() + totals = dict.fromkeys( + ( + "input_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + "output_tokens", + ), + 0, + ) + run(totals) + if len(NUMBERS) > 1 and totals["output_tokens"]: + print( + f"\ntotal: {totals['input_tokens']} in," + f" {totals['cache_creation_input_tokens']} written," + f" {totals['cache_read_input_tokens']} cached," + f" {totals['output_tokens']} out" + ) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000000..3a5b7331bf --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,134 @@ +# Generates wheels for all platforms, architectures and free-threaded builds, +# then checks the resulting distribution. Tests are run by tests.yml. +# +# Useful URLs: +# * https://github.com/pypa/cibuildwheel +# * https://github.com/astral-sh/setup-uv +# * https://github.com/docker/setup-qemu-action +# * https://github.com/actions/cache +# * https://github.com/actions/checkout +# * https://github.com/actions/download-artifact +# * https://github.com/actions/setup-python +# * https://github.com/actions/upload-artifact + +on: + workflow_dispatch: + push: + paths-ignore: + - "docs/**" + pull_request: + paths-ignore: + - "docs/**" +name: wheels +concurrency: + # Cancel run if a new one starts, but don't interrupt all jobs on the first + # failure. + group: wheels-${{ github.ref }} + cancel-in-progress: true +jobs: + wheels: + name: "${{ matrix.osname }} ${{ matrix.archname || matrix.arch }}${{ matrix.tag && format(' ({0})', matrix.tag) || '' }}" + # Skip same-repo PR runs: the push event already covers them. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + # We produce an abi3 wheel which works from Python 3.8 onwards. We + # generate it on Python 3.14, which is faster. + matrix: + include: + # manylinux and musllinux share a runner, so give them a lane each + # instead of building one set after the other. + - { os: ubuntu-latest, osname: linux, arch: x86_64, build: "cp314{,t}-manylinux*", tag: manylinux } + - { os: ubuntu-latest, osname: linux, arch: x86_64, build: "cp314-musllinux*", tag: musllinux } + - { os: ubuntu-24.04-arm, osname: linux, arch: aarch64, build: "cp314{,t}-manylinux*", tag: manylinux } + - { os: ubuntu-24.04-arm, osname: linux, arch: aarch64, build: "cp314-musllinux*", tag: musllinux } + - { os: ubuntu-latest, osname: linux, arch: ppc64le, build: "cp314-manylinux*", tag: abi3, qemu: ppc64le } + - { os: ubuntu-latest, osname: linux, arch: s390x, build: "cp314-manylinux*", tag: abi3, qemu: s390x } + # Cross-builds x86_64 under Rosetta, so one runner covers both arches + # (faster). + - { os: macos-15, osname: macos, arch: "arm64 x86_64", archname: both, build: "cp314-*", tag: abi3 } + - { os: macos-15, osname: macos, arch: "arm64 x86_64", archname: both, build: "cp314t-*", tag: cp314t } + # Same idea for Windows. + - { os: windows-2025, osname: win, arch: AMD64, build: "cp314-*", tag: abi3 } + - { os: windows-2025, osname: win, arch: AMD64, build: "cp314t-*", tag: cp314t } + - { os: windows-11-arm, osname: win, arch: ARM64, build: "cp314-*", tag: abi3 } + - { os: windows-11-arm, osname: win, arch: ARM64, build: "cp314t-*", tag: cp314t } + steps: + - uses: actions/checkout@v7 + + - name: Set up QEMU + if: matrix.qemu + uses: docker/setup-qemu-action@v4 + with: + platforms: ${{ matrix.qemu }} + + # Cache the CPython interpreters cibuildwheel downloads, so later pushes + # reuse them. Windows only: Linux takes them from the container, and + # macOS installs a .pkg into /Library/Frameworks, leaving nothing behind + # to cache. Worth ~90 secs on Windows, measured by removing it (see + # #2931). + - name: Cache cibuildwheel interpreters + if: runner.os == 'Windows' + uses: actions/cache@v6 + with: + path: ${{ runner.temp }}/cibw-cache + # Refresh the cache every time wheels.yml or pyproject.toml change. + # restore-keys falls back to the previous entry meanwhile, so an + # unrelated edit doesn't mean re-downloading everything. + key: cibw-${{ runner.os }}-${{ matrix.arch }}-${{ matrix.tag || 'all' }}-${{ hashFiles('.github/workflows/wheels.yml', 'pyproject.toml') }} + restore-keys: cibw-${{ runner.os }}-${{ matrix.arch }}-${{ matrix.tag || 'all' }}- + + - name: Install uv + uses: astral-sh/setup-uv@v9.0.0 + with: + version: "0.12.3" + + # Not pypa/cibuildwheel: that action pip-installs itself from an sdist on + # every lane, which takes ~18 secs instead of ~5. + - name: Build wheels + shell: bash + run: uvx --from 'cibuildwheel==4.1.1' cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: "${{ matrix.arch }}" + CIBW_BUILD: "${{ matrix.build }}" + CIBW_CACHE_PATH: ${{ runner.temp }}/cibw-cache + CIBW_BUILD_FRONTEND: "build[uv]" + CIBW_TEST_COMMAND: 'python -W error::RuntimeWarning -c "import psutil; print(psutil.__version__)"' + # Disabled, "make ci-check-dist" does it already. + CIBW_AUDIT_COMMAND: "" + # Pin it, else it follows whichever Python builds the wheel. + CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=10.15 + # We only link Windows system DLLs, so there's nothing for delvewheel + # to bundle. + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "" + + - name: Upload wheels + uses: actions/upload-artifact@v7 + with: + name: wheels-${{ matrix.os }}-${{ matrix.archname || matrix.arch }}-${{ matrix.tag || 'all' }} + path: wheelhouse + compression-level: 0 + + # Merge wheels and check sanity of the package distribution. + check-dist: + needs: [wheels] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: 3.x + - uses: actions/download-artifact@v8 + with: + pattern: wheels-* + path: wheelhouse + merge-multiple: true + - run: | + make ci-check-dist + - uses: actions/upload-artifact@v7 + with: + name: wheels + path: dist/*.whl + compression-level: 0 diff --git a/.gitignore b/.gitignore index 99d0d54571..3d8f05eaff 100644 --- a/.gitignore +++ b/.gitignore @@ -11,8 +11,12 @@ syntax: glob *.rej *.so *.swp +.failed-tests.txt .cache/ .idea/ .tox/ build/ +docs/_build/ dist/ +wheelhouse/ +.tests/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 17206c58a5..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,39 +0,0 @@ -sudo: false -language: python -cache: pip -matrix: - include: - - python: 2.6 - - python: 2.7 - - python: 3.3 - - python: 3.4 - - python: 3.5 - - "pypy" - # XXX - commented because OSX builds are deadly slow - # - language: generic - # os: osx - # env: PYVER=py26 - - language: generic - os: osx - env: PYVER=py27 - # XXX - commented because OSX builds are deadly slow - # - language: generic - # os: osx - # env: PYVER=py33 - - language: generic - os: osx - env: PYVER=py34 - # XXX - not supported yet - # - language: generic - # os: osx - # env: PYVER=py35 -install: - - ./.ci/travis/install.sh -script: - - ./.ci/travis/run.sh -after_success: - # upload reports to coveralls.io - - | - if [ "$(uname -s)" != 'Darwin' ]; then - coveralls - fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..b704d7d3b4 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing to psutil project + +## Issues + +- The issue tracker is for reporting problems or proposing enhancements related + to the **program code**. +- Please do not open issues **asking for support**. Instead, use the forum at: + https://groups.google.com/g/psutil. +- Before submitting a new issue, **search** if there are existing issues for + the same topic. +- **Be clear** in describing what the problem is, and fill in the default issue + **template**. There is a bot which reads the title and description and + assigns **labels** automatically. Labels help keeping the issues properly + organized and searchable (by OS, issue type, etc.). +- When reporting a malfunction, consider enabling + [debug mode](https://psutil.io/devguide/#debug-mode) first. +- To report a **security vulnerability**, use the + [Tidelift security contact](https://tidelift.com/security). Tidelift will + coordinate the fix and the disclosure of the reported problem. + +## Pull Requests + +- In order to get acquainted with the code base and tooling, take a look at the + **[Development Guide](https://psutil.io/devguide/)**. +- The PR system is for fixing bugs or make enhancements related to the + **program code**. +- If you wish to implement a new feature or add support for a new platform it's + better to **discuss it first**, either on the issue tracker, the forum or via + private email. +- No need to touch the changelog or credits files. A bot automatically adds + those entries for you before merging. diff --git a/CREDITS b/CREDITS deleted file mode 100644 index ab555efd9e..0000000000 --- a/CREDITS +++ /dev/null @@ -1,400 +0,0 @@ -Intro -===== - -I would like to recognize some of the people who have been instrumental in the -development of psutil. -I'm sure I'm forgetting some people (feel free to email me), but here is a -short list. -It's modeled after the Linux CREDITS file where the fields are: -name (N), e-mail (E), web-address (W), country (C), description (D), (I) issues -(issue tracker is at https://github.com/giampaolo/psutil/issues). -Really thanks to all of you. - -- Giampaolo - -Author -====== - -N: Giampaolo Rodola' -C: Italy -E: g.rodola@gmail.com -W: http://grodola.blogspot.com/ - -Contributors -============ - -N: Jay Loden -C: NJ, USA -E: jloden@gmail.com -D: original co-author, initial design/bootstrap and occasional bug fixes -W: http://www.jayloden.com - -N: Jeff Tang -W: https://github.com/mrjefftang -I: 340, 529, 616, 653, 654, 648, 641 - -N: Jeremy Whitlock -E: jcscoobyrs@gmail.com -D: great help with OSX C development. -I: 125, 150, 174, 206 - -N: Landry Breuil -W: https://github.com/landryb -D: OpenBSD implementation. -I: 615 - -N: wj32 -E: wj32.64@gmail.com -D: process username() and get_connections() on Windows -I: 114, 115 - -N: Yan Raber -C: Bologna, Italy -E: yanraber@gmail.com -D: help on Windows development (initial version of Process.username()) - -N: Justin Venus -E: justin.venus@gmail.com -D: Solaris support -I: 18 - -N: Dave Daeschler -C: USA -E: david.daeschler@gmail.com -W: http://daviddaeschler.com -D: some contributions to initial design/bootstrap plus occasional bug fixing -I: 522, 536 - -N: Thomas Klausner -W: https://github.com/0-wiz-0 -I: #557 - -N: Ryo Onodera -W: https://github.com/ryoon -I: #557 - -N: cjgohlke -E: cjgohlke@gmail.com -D: Windows 64 bit support -I: 107 - -N: Jeffery Kline -E: jeffery.kline@gmail.com -I: 130 - -N: Grabriel Monnerat -E: gabrielmonnerat@gmail.com -I: 146 - -N: Philip Roberts -E: philip.roberts@gmail.com -I: 168 - -N: jcscoobyrs -E: jcscoobyrs@gmail.com -I: 125 - -N: Sandro Tosi -E: sandro.tosi@gmail.com -I: 200, 201 - -N: Andrew Colin -E: andrew.colin@gmail.com -I: 248 - -N: Amoser -E: amoser@google.com -I: 266, 267, 340 - -N: Matthew Grant -E: matthewgrant5@gmail.com -I: 271 - -N: oweidner -E: oweidner@cct.lsu.edu -I: 275 - -N: Tarek Ziade -E: ziade.tarek -I: 281 - -N: Luca Cipriani -C: Turin, Italy -E: luca.opensource@gmail.com -I: 278 - -N: Maciej Lach, -E: maciej.lach@gmail.com -I: 294 - -N: James Pye -E: james.pye@gmail.com -I: 305, 306 - -N: Stanchev Emil -E: stanchev.emil -I: 314 - -N: Kim Gräsman -E: kim.grasman@gmail.com -D: ...also kindly donated some money. -I: 316 - -N: Riccardo Murri -C: Italy -I: 318 - -N: Florent Xicluna -E: florent.xicluna@gmail.com -I: 319 - -N: Michal Spondr -E: michal.spondr -I: 313 - -N: Jean Sebastien -E: dumbboules@gmail.com -I: 344 - -N: Rob Smith -W: http://www.kormoc.com/ -I: 341 - -N: Youngsik Kim -W: https://plus.google.com/101320747613749824490/ -I: 317 - -N: Gregory Szorc -W: https://plus.google.com/116873264322260110710/posts -I: 323 - -N: André Oriani -E: aoriani@gmail.com -I: 361 - -N: clackwell -E: clackwell@gmail.com -I: 356 - -N: m.malycha -E: m.malycha@gmail.com -I: 351 - -N: John Baldwin -E: jhb@FreeBSD.org -I: 370 - -N: Jan Beich -E: jbeich@tormail.org -I: 325 - -N: floppymaster -E: floppymaster@gmail.com -I: 380 - -N: Arfrever.FTA -E: Arfrever.FTA@gmail.com -I: 369, 404 - -N: danudey -E: danudey@gmail.com -I: 386 - -N: Adrien Fallou -I: 224 - -N: Gisle Vanem -E: gisle.vanem@gmail.com -I: 411 - -N: thepyr0 -E: thepyr0@gmail.com -I: 414 - -N: John Pankov -E: john.pankov@gmail.com -I: 435 - -N: Matt Good -W: http://matt-good.net/ -I: 438 - -N: Ulrich Klank -E: ulrich.klank@scitics.de -I: 448 - -N: Josiah Carlson -E: josiah.carlson@gmail.com -I: 451, 452 - -N: Raymond Hettinger -D: namedtuple and lru_cache backward compatible implementations. - -N: Jason Kirtland -D: backward compatible implementation of collections.defaultdict. - -M: Ken Seeho -D: @cached_property decorator - -N: crusaderky -E: crusaderky@gmail.com -I: 470, 477 - -E: alex@mroja.net -I: 471 - -N: Gautam Singh -E: gautam.singh@gmail.com -I: 466 - -E: lhn@hupfeldtit.dk -I: 476, 479 - -N: Francois Charron -E: francois.charron.1@gmail.com -I: 474 - -N: Naveed Roudsari -E: naveed.roudsari@gmail.com -I: 421 - -N: Alexander Grothe -E: Alexander.Grothe@gmail.com -I: 497 - -N: Szigeti Gabor Niif -E: szigeti.gabor.niif@gmail.com -I: 446 - -N: msabramo -E: msabramo@gmail.com -I: 492 - -N: Yaolong Huang -E: airekans@gmail.com -W: http://airekans.github.io/ -I: 530 - -N: Anders Chrigström -W: https://github.com/anders-chrigstrom -I: 496 - -N: spacewander -W: https://github.com/spacewander -E: spacewanderlzx@gmail.com -I: 561, 603 - -N: Sylvain Mouquet -E: sylvain.mouquet@gmail.com -I: 565 - -N: karthikrev -I: 568 - -N: Bruno Binet -E: bruno.binet@gmail.com -I: 572 - -N: Gabi Davar -C: Israel -W: https://github.com/mindw -I: 578, 581, 587 - -N: spacewanderlzx -C: Guangzhou,China -E: spacewanderlzx@gmail.com -I: 555 - -N: Fabian Groffen -I: 611, 618 - -N: desbma -W: https://github.com/desbma -C: France -I: 628 - -N: John Burnett -W: http://www.johnburnett.com/ -C: Irvine, CA, US -I: 614 - -N: Ãrni Már Jónsson -E: Reykjavik, Iceland -E: https://github.com/arnimarj -I: 634 - -N: Bart van Kleef -W: https://github.com/bkleef -I: 664 - -N: Steven Winfield -W: https://github.com/stevenwinfield -I: 672 - -N: sk6249 -W: https://github.com/sk6249 -I: 670 - -N: maozguttman -W: https://github.com/maozguttman -I: 659 - -N: wiggin15 -W: https://github.com/wiggin15 -I: 517, 607, 610 - -N: dasumin -W: https://github.com/dasumin -I: 541 - -N: Mike Sarahan -W: https://github.com/msarahan -I: 688 - -N: Syohei YOSHIDA -W: https://github.com/syohex -I: 730 - -N: Frank Benkstein -W: https://github.com/fbenkstein -I: 732, 733 - -N: Visa Hankala -E: visa@openbsd.org -I: 741 - -N: Sebastian-Gabriel Brestin -C: Romania -E: sebastianbrestin@gmail.com -I: 704 - -N: Timmy Konick -W: https://github.com/tijko -I: 751 - -N: mpderbec -W: https://github.com/mpderbec -I: 660 - -N: Mozilla Foundation -D: sample code for process USS memory. - -N: wxwright -W: https://github.com/wxwright -I: 776 - -N: Farhan Khan -E: khanzf@gmail.com -I: 823 - -N: Jake Omann -E: https://github.com/jhomann -I: 816 - -N: Jeremy Humble -W: https://github.com/jhumble -I: 863 - -N: Ilya Georgievsky -W: https://github.com/xBeAsTx -I: 870 diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst deleted file mode 100644 index ebd919abd4..0000000000 --- a/DEVGUIDE.rst +++ /dev/null @@ -1,166 +0,0 @@ -===== -Setup -===== - -If you plan on hacking on psutil this is what you're supposed to do first: - -- clone the GIT repository:: - - $ git clone git@github.com:giampaolo/psutil.git - -- install system deps (see `install instructions `__). - -- install development deps; these are useful for running tests (e.g. mock, - unittest2), building doc (e.g. sphinx), running linters (flake8), etc. :: - - $ make setup-dev-env - -- bear in mind that ``make`` (see `Makefile `_) - is the designated tool to run tests, build, install etc. and that it is also - available on Windows - (see `make.bat `_). -- bear in mind that both psutil (``make install``) and any other lib - (``make setup-dev-env``) is installed as a limited user - (``pip install --user ...``), so develop as such (don't use root). -- (UNIX only) run ``make install-git-hooks``: this will reject your commits - if python code is not PEP8 compliant. -- run ``make test`` to run tests. - -============ -Coding style -============ - -- python code strictly follows `PEP 8 `_ - styling guides and this is enforced by ``make install-git-hooks``. -- C code strictly follows `PEP 7 `_ - styling guides. - -======== -Makefile -======== - -Some useful make commands:: - - $ make install # install - $ make setup-dev-env # install useful dev libs (pyflakes, unittest2, etc.) - $ make test # run all tests - $ make test-memleaks # run memory leak tests - $ make coverage # run test coverage - $ make flake8 # run PEP8 linter - -==================== -Adding a new feature -==================== - -Usually the files involved when adding a new functionality are: - -.. code-block:: plain - - psutil/__init__.py # main psutil namespace - psutil/_ps{platform}.py # python platform wrapper - psutil/_psutil_{platform}.c # C platform extension - psutil/_psutil_{platform}.h # C header file - psutil/tests/test_process|system.py # main test suite - psutil/tests/test_{platform}.py # platform specific test suite - -Typical process occurring when adding a new functionality (API): - -- define the new function in ``psutil/__init__.py``. -- write the platform specific implementation in ``psutil/_ps{platform}.py`` - (e.g. ``psutil/_pslinux.py``). -- if the change requires C, write the C implementation in - ``psutil/_psutil_{platform}.c`` (e.g. ``psutil/_psutil_linux.c``). -- write a generic test in ``psutil/tests/test_system.py`` or - ``psutil/tests/test_process.py``. -- if possible, write a platform specific test in - ``psutil/tests/test_{platform}.py`` (e.g. ``test_linux.py``). - This usually means testing the return value of the new feature against - a system CLI tool. -- update doc in ``doc/index.py``. -- update ``HISTORY.rst``. -- update ``README.rst`` (if necessary). -- make a pull request. - -====================== -Continuous integration -====================== - -All of the services listed below are automatically run on ``git push``. - -Unit tests ----------- - -Tests are automatically run for every GIT push on **Linux**, **OSX** and -**Windows** by using: - -- `Travis `_ (Linux, OSX) -- `Appveyor `_ (Windows) - -Test files controlling these are -`.travis.yml `_ -and -`appveyor.yml `_. -Both services run psutil test suite against all supported python version -(2.6 - 3.5). -Two icons in the home page (README) always show the build status: - -.. image:: https://api.travis-ci.org/giampaolo/psutil.png?branch=master - :target: https://travis-ci.org/giampaolo/psutil - :alt: Linux tests (Travis) - -.. image:: https://ci.appveyor.com/api/projects/status/qdwvw7v1t915ywr5/branch/master?svg=true - :target: https://ci.appveyor.com/project/giampaolo/psutil - :alt: Windows tests (Appveyor) - -OSX, FreeBSD and Solaris are currently tested manually (sigh!). - -Test coverage -------------- - -Test coverage is provided by `coveralls.io `_, -it is controlled via `.travis.yml `_ -and it is updated on every git push. -An icon in the home page (README) always shows the last coverage percentage: - -.. image:: https://coveralls.io/repos/giampaolo/psutil/badge.svg?branch=master&service=github - :target: https://coveralls.io/github/giampaolo/psutil?branch=master - :alt: Test coverage (coverall.io) - -============= -Documentation -============= - -- doc source code is written in a single file: `/docs/index.rst `_. -- it uses `RsT syntax `_ - and it's built with `sphinx `_. -- doc can be built with ``make setup-dev-env; cd docs; make html``. -- public doc is hosted on http://pythonhosted.org/psutil/. -- it is uploaded on every new release with ``make upload-doc``. - -======================= -Releasing a new version -======================= - -These are note for myself (Giampaolo): - -- make sure all tests pass and all builds are green. -- upload source tarball on PYPI with ``make upload-src``. -- upload exe and wheel files for windows on PYPI with ``make upload-all``. - - ...or by using atrifacts hosted on AppVeyor with ``make win-download-exes`` - and ``make win-upload-exes``, -- upload updated doc on http://pythonhosted.org/psutil with ``make upload-doc``. -- GIT tag the new release with ``make git-tag-release``. -- post on psutil and python-announce mailing lists, twitter, g+, blog. - -============= -FreeBSD notes -============= - -- setup: - -.. code-block:: bash - - $ pkg install python python3 gcc git vim screen bash - $ chsh -s /usr/local/bin/bash user # set bash as default shell - -- ``/usr/src`` contains the source codes for all installed CLI tools (grep in it). diff --git a/HISTORY.rst b/HISTORY.rst index bb7162868e..a5621dc086 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,1255 +1,3 @@ -Bug tracker at https://github.com/giampaolo/psutil/issues +History has moved to: -4.3.1 - XXXX-XX-XX -================== - -**Bug fixes** - -- #854: Process.as_dict() raises ValueError if passed an erroneous attrs name. -- #857: [SunOS] Process cpu_times(), cpu_percent(), threads() amd memory_maps() - may raise RuntimeError if attempting to query a 64bit process with a 32bit - python. "Null" values are returned as a fallback. -- #858: Process.as_dict() should not return memory_info_ex() because it's - deprecated. -- #863: [Windows] memory_map truncates addresses above 32 bits -- #866: [Windows] win_service_iter() and services in general are not able to - handle unicode service names / descriptions. -- #869: [Windows] Process.wait() may raise TimeoutExpired with wrong timeout - unit (ms instead of sec). -- #870: [Windows] Handle leak inside psutil_get_process_data. - - -4.3.0 - 2016-06-18 -================== - -**Enhancements** - -- #819: [Linux] different speedup improvements: - Process.ppid() is 20% faster - Process.status() is 28% faster - Process.name() is 25% faster - Process.num_threads is 20% faster on Python 3 - -**Bug fixes** - -- #810: [Windows] Windows wheels are incompatible with pip 7.1.2. -- #812: [NetBSD] fix compilation on NetBSD-5.x. -- #823: [NetBSD] virtual_memory() raises TypeError on Python 3. -- #829: [UNIX] psutil.disk_usage() percent field takes root reserved space - into account. -- #816: [Windows] fixed net_io_counter() values wrapping after 4.3GB in - Windows Vista (NT 6.0) and above using 64bit values from newer win APIs. - - -4.2.0 - 2016-05-14 -================== - -**Enhancements** - -- #795: [Windows] new APIs to deal with Windows services: win_service_iter() - and win_service_get(). -- #800: [Linux] psutil.virtual_memory() returns a new "shared" memory field. -- #819: [Linux] speedup /proc parsing: - - Process.ppid() is 20% faster - - Process.status() is 28% faster - - Process.name() is 25% faster - - Process.num_threads is 20% faster on Python 3 - -**Bug fixes** - -- #797: [Linux] net_if_stats() may raise OSError for certain NIC cards. -- #813: Process.as_dict() should ignore extraneous attribute names which gets - attached to the Process instance. - - -4.1.0 - 2016-03-12 -================== - -**Enhancements** - -- #777: [Linux] Process.open_files() on Linux return 3 new fields: position, - mode and flags. -- #779: Process.cpu_times() returns two new fields, 'children_user' and - 'children_system' (always set to 0 on OSX and Windows). -- #789: [Windows] psutil.cpu_times() return two new fields: "interrupt" and - "dpc". Same for psutil.cpu_times_percent(). -- #792: new psutil.cpu_stats() function returning number of CPU ctx switches - interrupts, soft interrupts and syscalls. - -**Bug fixes** - -- #774: [FreeBSD] net_io_counters() dropout is no longer set to 0 if the kernel - provides it. -- #776: [Linux] Process.cpu_affinity() may erroneously raise NoSuchProcess. - (patch by wxwright) -- #780: [OSX] psutil does not compile with some gcc versions. -- #786: net_if_addrs() may report incomplete MAC addresses. -- #788: [NetBSD] virtual_memory()'s buffers and shared values were set to 0. -- #790: [OSX] psutil won't compile on OSX 10.4. - - -4.0.0 - 2016-02-17 -================== - -**Enhancements** - -- #523: [Linux, FreeBSD] disk_io_counters() return a new "busy_time" field. -- #660: [Windows] make.bat is smarter in finding alternative VS install - locations. (patch by mpderbec) -- #732: Process.environ(). (patch by Frank Benkstein) -- #753: [Linux, OSX, Windows] Process USS and PSS (Linux) "real" memory stats. - (patch by Eric Rahm) -- #755: Process.memory_percent() "memtype" parameter. -- #758: tests now live in psutil namespace. -- #760: expose OS constants (psutil.LINUX, psutil.OSX, etc.) -- #756: [Linux] disk_io_counters() return 2 new fields: read_merged_count and - write_merged_count. -- #762: new scripts/procsmem.py script. - -**Bug fixes** - -- #685: [Linux] virtual_memory() provides wrong results on systems with a lot - of physical memory. -- #704: [Solaris] psutil does not compile on Solaris sparc. -- #734: on Python 3 invalid UTF-8 data is not correctly handled for process - name(), cwd(), exe(), cmdline() and open_files() methods resulting in - UnicodeDecodeError exceptions. 'surrogateescape' error handler is now - used as a workaround for replacing the corrupted data. -- #737: [Windows] when the bitness of psutil and the target process was - different cmdline() and cwd() could return a wrong result or incorrectly - report an AccessDenied error. -- #741: [OpenBSD] psutil does not compile on mips64. -- #751: [Linux] fixed call to Py_DECREF on possible Null object. -- #754: [Linux] cmdline() can be wrong in case of zombie process. -- #759: [Linux] Process.memory_maps() may return paths ending with " (deleted)" -- #761: [Windows] psutil.boot_time() wraps to 0 after 49 days. -- #764: [NetBSD] fix compilation on NetBSD-6.x. -- #766: [Linux] net_connections() can't handle malformed /proc/net/unix file. -- #767: [Linux] disk_io_counters() may raise ValueError on 2.6 kernels and it's - broken on 2.4 kernels. -- #770: [NetBSD] disk_io_counters() metrics didn't update. - - -3.4.2 - 2016-01-20 -================== - -**Enhancements** - -- #728: [Solaris] exposed psutil.PROCFS_PATH constant to change the default - location of /proc filesystem. - -**Bug fixes** - -- #724: [FreeBSD] psutil.virtual_memory().total is incorrect. -- #730: [FreeBSD] psutil.virtual_memory() crashes. - - -3.4.1 - 2016-01-15 -================== - -**Enhancements** - -- #557: [NetBSD] added NetBSD support. (contributed by Ryo Onodera and - Thomas Klausner) -- #708: [Linux] psutil.net_connections() and Process.connections() on Python 2 - can be up to 3x faster in case of many connections. - Also psutil.Process.memory_maps() is slightly faster. -- #718: process_iter() is now thread safe. - -**Bug fixes** - -- #714: [OpenBSD] virtual_memory().cached value was always set to 0. -- #715: don't crash at import time if cpu_times() fail for some reason. -- #717: [Linux] Process.open_files fails if deleted files still visible. -- #722: [Linux] swap_memory() no longer crashes if sin/sout can't be determined - due to missing /proc/vmstat. -- #724: [FreeBSD] virtual_memory().total is slightly incorrect. - - -3.3.0 - 2015-11-25 -================== - -**Enhancements** - -- #558: [Linux] exposed psutil.PROCFS_PATH constant to change the default - location of /proc filesystem. -- #615: [OpenBSD] added OpenBSD support. (contributed by Landry Breuil) - -**Bug fixes** - -- #692: [UNIX] Process.name() is no longer cached as it may change. - - -3.2.2 - 2015-10-04 -================== - -**Bug fixes** - -- #517: [SunOS] net_io_counters failed to detect network interfaces - correctly on Solaris 10 -- #541: [FreeBSD] disk_io_counters r/w times were expressed in seconds instead - of milliseconds. (patch by dasumin) -- #610: [SunOS] fix build and tests on Solaris 10 -- #623: [Linux] process or system connections raises ValueError if IPv6 is not - supported by the system. -- #678: [Linux] can't install psutil due to bug in setup.py. -- #688: [Windows] compilation fails with MSVC 2015, Python 3.5. (patch by - Mike Sarahan) - - -3.2.1 - 2015-09-03 -================== - -**Bug fixes** - -- #677: [Linux] can't install psutil due to bug in setup.py. - - -3.2.0 - 2015-09-02 -================== - -**Enhancements** - -- #644: [Windows] added support for CTRL_C_EVENT and CTRL_BREAK_EVENT signals - to use with Process.send_signal(). -- #648: CI test integration for OSX. (patch by Jeff Tang) -- #663: [UNIX] net_if_addrs() now returns point-to-point (VPNs) addresses. -- #655: [Windows] different issues regarding unicode handling were fixed. On - Python 2 all APIs returning a string will now return an encoded version of it - by using sys.getfilesystemencoding() codec. The APIs involved are: - - psutil.net_if_addrs() - - psutil.net_if_stats() - - psutil.net_io_counters() - - psutil.Process.cmdline() - - psutil.Process.name() - - psutil.Process.username() - - psutil.users() - -**Bug fixes** - -- #513: [Linux] fixed integer overflow for RLIM_INFINITY. -- #641: [Windows] fixed many compilation warnings. (patch by Jeff Tang) -- #652: [Windows] net_if_addrs() UnicodeDecodeError in case of non-ASCII NIC - names. -- #655: [Windows] net_if_stats() UnicodeDecodeError in case of non-ASCII NIC - names. -- #659: [Linux] compilation error on Suse 10. (patch by maozguttman) -- #664: [Linux] compilation error on Alpine Linux. (patch by Bart van Kleef) -- #670: [Windows] segfgault of net_if_addrs() in case of non-ASCII NIC names. - (patch by sk6249) -- #672: [Windows] compilation fails if using Windows SDK v8.0. (patch by - Steven Winfield) -- #675: [Linux] net_connections(); UnicodeDecodeError may occur when listing - UNIX sockets. - - -3.1.1 - 2015-07-15 -================== - -**Bug fixes** - -- #603: [Linux] ionice_set value range is incorrect. (patch by spacewander) -- #645: [Linux] psutil.cpu_times_percent() may produce negative results. -- #656: 'from psutil import *' does not work. - - -3.1.0 - 2015-07-15 -================== - -**Enhancements** - -- #534: [Linux] disk_partitions() added support for ZFS filesystems. -- #646: continuous tests integration for Windows with - https://ci.appveyor.com/project/giampaolo/psutil. -- #647: new dev guide: - https://github.com/giampaolo/psutil/blob/master/DEVGUIDE.rst -- #651: continuous code quality test integration with - https://scrutinizer-ci.com/g/giampaolo/psutil/ - -**Bug fixes** - -- #340: [Windows] Process.open_files() no longer hangs. Instead it uses a - thred which times out and skips the file handle in case it's taking too long - to be retrieved. (patch by Jeff Tang, PR #597) -- #627: [Windows] Process.name() no longer raises AccessDenied for pids owned - by another user. -- #636: [Windows] Process.memory_info() raise AccessDenied. -- #637: [UNIX] raise exception if trying to send signal to Process PID 0 as it - will affect os.getpid()'s process group instead of PID 0. -- #639: [Linux] Process.cmdline() can be truncated. -- #640: [Linux] *connections functions may swallow errors and return an - incomplete list of connnections. -- #642: repr() of exceptions is incorrect. -- #653: [Windows] Add inet_ntop function for Windows XP to support IPv6. -- #641: [Windows] Replace deprecated string functions with safe equivalents. - - -3.0.1 - 2015-06-18 -================== - -**Bug fixes** - -- #632: [Linux] better error message if cannot parse process UNIX connections. -- #634: [Linux] Proces.cmdline() does not include empty string arguments. -- #635: [UNIX] crash on module import if 'enum' package is installed on python - < 3.4. - - -3.0.0 - 2015-06-13 -================== - -**Enhancements** - -- #250: new psutil.net_if_stats() returning NIC statistics (isup, duplex, - speed, MTU). -- #376: new psutil.net_if_addrs() returning all NIC addresses a-la ifconfig. -- #469: on Python >= 3.4 ``IOPRIO_CLASS_*`` and ``*_PRIORITY_CLASS`` constants - returned by psutil.Process' ionice() and nice() methods are enums instead of - plain integers. -- #581: add .gitignore. (patch by Gabi Davar) -- #582: connection constants returned by psutil.net_connections() and - psutil.Process.connections() were turned from int to enums on Python > 3.4. -- #587: Move native extension into the package. -- #589: Process.cpu_affinity() accepts any kind of iterable (set, tuple, ...), - not only lists. -- #594: all deprecated APIs were removed. -- #599: [Windows] process name() can now be determined for all processes even - when running as a limited user. -- #602: pre-commit GIT hook. -- #629: enhanced support for py.test and nose test discovery and tests run. -- #616: [Windows] Add inet_ntop function for Windows XP. - -**Bug fixes** - -- #428: [all UNIXes except Linux] correct handling of zombie processes; - introduced new ZombieProcess exception class. -- #512: [BSD] fix segfault in net_connections(). -- #555: [Linux] psutil.users() correctly handles ":0" as an alias for - "localhost" -- #579: [Windows] Fixed open_files() for PID>64K. -- #579: [Windows] fixed many compiler warnings. -- #585: [FreeBSD] net_connections() may raise KeyError. -- #586: [FreeBSD] cpu_affinity() segfaults on set in case an invalid CPU - number is provided. -- #593: [FreeBSD] Process().memory_maps() segfaults. -- #606: Process.parent() may swallow NoSuchProcess exceptions. -- #611: [SunOS] net_io_counters has send and received swapped -- #614: [Linux]: cpu_count(logical=False) return the number of physical CPUs - instead of physical cores. -- #618: [SunOS] swap tests fail on Solaris when run as normal user -- #628: [Linux] Process.name() truncates process name in case it contains - spaces or parentheses. - - -2.2.1 - 2015-02-02 -================== - -**Bug fixes** - -- #496: [Linux] fix "ValueError: ambiguos inode with multiple PIDs references" - (patch by Bruno Binet) - - -2.2.0 - 2015-01-06 -================== - -**Enhancements** - -- #521: drop support for Python 2.4 and 2.5. -- #553: new examples/pstree.py script. -- #564: C extension version mismatch in case the user messed up with psutil - installation or with sys.path is now detected at import time. -- #568: New examples/pidof.py script. -- #569: [FreeBSD] add support for process CPU affinity. - -**Bug fixes** - -- #496: [Solaris] can't import psutil. -- #547: [UNIX] Process.username() may raise KeyError if UID can't be resolved. -- #551: [Windows] get rid of the unicode hack for net_io_counters() NIC names. -- #556: [Linux] lots of file handles were left open. -- #561: [Linux] net_connections() might skip some legitimate UNIX sockets. - (patch by spacewander) -- #565: [Windows] use proper encoding for psutil.Process.username() and - psutil.users(). (patch by Sylvain Mouquet) -- #567: [Linux] in the alternative implementation of CPU affinity PyList_Append - and Py_BuildValue return values are not checked. -- #569: [FreeBSD] fix memory leak in psutil.cpu_count(logical=False). -- #571: [Linux] Process.open_files() might swallow AccessDenied exceptions and - return an incomplete list of open files. - - -2.1.3 - 2014-09-26 -================== - -- #536: [Linux]: fix "undefined symbol: CPU_ALLOC" compilation error. - - -2.1.2 - 2014-09-21 -================== - -**Enhancements** - -- #407: project moved from Google Code to Github; code moved from Mercurial - to Git. -- #492: use tox to run tests on multiple python versions. (patch by msabramo) -- #505: [Windows] distribution as wheel packages. -- #511: new examples/ps.py sample code. - -**Bug fixes** - -- #340: [Windows] Process.get_open_files() no longer hangs. (patch by - Jeff Tang) -- #501: [Windows] disk_io_counters() may return negative values. -- #503: [Linux] in rare conditions Process exe(), open_files() and - connections() methods can raise OSError(ESRCH) instead of NoSuchProcess. -- #504: [Linux] can't build RPM packages via setup.py -- #506: [Linux] python 2.4 support was broken. -- #522: [Linux] Process.cpu_affinity() might return EINVAL. (patch by David - Daeschler) -- #529: [Windows] Process.exe() may raise unhandled WindowsError exception - for PIDs 0 and 4. (patch by Jeff Tang) -- #530: [Linux] psutil.disk_io_counters() may crash on old Linux distros - (< 2.6.5) (patch by Yaolong Huang) -- #533: [Linux] Process.memory_maps() may raise TypeError on old Linux distros. - - -2.1.1 - 2014-04-30 -================== - -**Bug fixes** - -- #446: [Windows] fix encoding error when using net_io_counters() on Python 3. - (patch by Szigeti Gabor Niif) -- #460: [Windows] net_io_counters() wraps after 4G. -- #491: [Linux] psutil.net_connections() exceptions. (patch by Alexander Grothe) - - -2.1.0 - 2014-04-08 -================== - -**Enhancements** - -- #387: system-wide open connections a-la netstat. - -**Bug fixes** - -- #421: [Solaris] psutil does not compile on SunOS 5.10 (patch by Naveed - Roudsari) -- #489: [Linux] psutil.disk_partitions() return an empty list. - - -2.0.0 - 2014-03-10 -================== - -**Enhancements** - -- #424: [Windows] installer for Python 3.X 64 bit. -- #427: number of logical and physical CPUs (psutil.cpu_count()). -- #447: psutil.wait_procs() timeout parameter is now optional. -- #452: make Process instances hashable and usable with set()s. -- #453: tests on Python < 2.7 require unittest2 module. -- #459: add a make file for running tests and other repetitive tasks (also - on Windows). -- #463: make timeout parameter of cpu_percent* functions default to 0.0 'cause - it's a common trap to introduce slowdowns. -- #468: move documentation to readthedocs.com. -- #477: process cpu_percent() is about 30% faster. (suggested by crusaderky) -- #478: [Linux] almost all APIs are about 30% faster on Python 3.X. -- #479: long deprecated psutil.error module is gone; exception classes now - live in "psutil" namespace only. - -**Bug fixes** - -- #193: psutil.Popen constructor can throw an exception if the spawned process - terminates quickly. -- #340: [Windows] process get_open_files() no longer hangs. (patch by - jtang@vahna.net) -- #443: [Linux] fix a potential overflow issue for Process.set_cpu_affinity() - on systems with more than 64 CPUs. -- #448: [Windows] get_children() and ppid() memory leak (patch by Ulrich - Klank). -- #457: [POSIX] pid_exists() always returns True for PID 0. -- #461: namedtuples are not pickle-able. -- #466: [Linux] process exe improper null bytes handling. (patch by - Gautam Singh) -- #470: wait_procs() might not wait. (patch by crusaderky) -- #471: [Windows] process exe improper unicode handling. (patch by - alex@mroja.net) -- #473: psutil.Popen.wait() does not set returncode attribute. -- #474: [Windows] Process.cpu_percent() is no longer capped at 100%. -- #476: [Linux] encoding error for process name and cmdline. - -**API changes** - -For the sake of consistency a lot of psutil APIs have been renamed. -In most cases accessing the old names will work but it will cause a -DeprecationWarning. - -- psutil.* module level constants have being replaced by functions: - - +-----------------------+-------------------------------+ - | Old name | Replacement | - +=======================+===============================+ - | psutil.NUM_CPUS | psutil.cpu_cpunt() | - +-----------------------+-------------------------------+ - | psutil.BOOT_TIME | psutil.boot_time() | - +-----------------------+-------------------------------+ - | psutil.TOTAL_PHYMEM | psutil.virtual_memory().total | - +-----------------------+-------------------------------+ - -- Renamed psutil.* functions: - - +--------------------------+-------------------------------+ - | Old name | Replacement | - +==========================+===============================+ - | - psutil.get_pid_list() | psutil.pids() | - +--------------------------+-------------------------------+ - | - psutil.get_users() | psutil.users() | - +--------------------------+-------------------------------+ - | - psutil.get_boot_time() | psutil.boot_time() | - +--------------------------+-------------------------------+ - -- All psutil.Process ``get_*`` methods lost the ``get_`` prefix. - get_ext_memory_info() renamed to memory_info_ex(). - Assuming "p = psutil.Process()": - - +--------------------------+----------------------+ - | Old name | Replacement | - +==========================+======================+ - | p.get_children() | p.children() | - +--------------------------+----------------------+ - | p.get_connections() | p.connections() | - +--------------------------+----------------------+ - | p.get_cpu_affinity() | p.cpu_affinity() | - +--------------------------+----------------------+ - | p.get_cpu_percent() | p.cpu_percent() | - +--------------------------+----------------------+ - | p.get_cpu_times() | p.cpu_times() | - +--------------------------+----------------------+ - | p.get_ext_memory_info() | p.memory_info_ex() | - +--------------------------+----------------------+ - | p.get_io_counters() | p.io_counters() | - +--------------------------+----------------------+ - | p.get_ionice() | p.ionice() | - +--------------------------+----------------------+ - | p.get_memory_info() | p.memory_info() | - +--------------------------+----------------------+ - | p.get_memory_maps() | p.memory_maps() | - +--------------------------+----------------------+ - | p.get_memory_percent() | p.memory_percent() | - +--------------------------+----------------------+ - | p.get_nice() | p.nice() | - +--------------------------+----------------------+ - | p.get_num_ctx_switches() | p.num_ctx_switches() | - +--------------------------+----------------------+ - | p.get_num_fds() | p.num_fds() | - +--------------------------+----------------------+ - | p.get_num_threads() | p.num_threads() | - +--------------------------+----------------------+ - | p.get_open_files() | p.open_files() | - +--------------------------+----------------------+ - | p.get_rlimit() | p.rlimit() | - +--------------------------+----------------------+ - | p.get_threads() | p.threads() | - +--------------------------+----------------------+ - | p.getcwd() | p.cwd() | - +--------------------------+----------------------+ - -- All psutil.Process ``set_*`` methods lost the ``set_`` prefix. - Assuming "p = psutil.Process()": - - +----------------------+---------------------------------+ - | Old name | Replacement | - +======================+=================================+ - | p.set_nice() | p.nice(value) | - +----------------------+---------------------------------+ - | p.set_ionice() | p.ionice(ioclass, value=None) | - +----------------------+---------------------------------+ - | p.set_cpu_affinity() | p.cpu_affinity(cpus) | - +----------------------+---------------------------------+ - | p.set_rlimit() | p.rlimit(resource, limits=None) | - +----------------------+---------------------------------+ - -- Except for 'pid' all psutil.Process class properties have been turned into - methods. This is the only case which there are no aliases. - Assuming "p = psutil.Process()": - - +---------------+-----------------+ - | Old name | Replacement | - +===============+=================+ - | p.name | p.name() | - +---------------+-----------------+ - | p.parent | p.parent() | - +---------------+-----------------+ - | p.ppid | p.ppid() | - +---------------+-----------------+ - | p.exe | p.exe() | - +---------------+-----------------+ - | p.cmdline | p.cmdline() | - +---------------+-----------------+ - | p.status | p.status() | - +---------------+-----------------+ - | p.uids | p.uids() | - +---------------+-----------------+ - | p.gids | p.gids() | - +---------------+-----------------+ - | p.username | p.username() | - +---------------+-----------------+ - | p.create_time | p.create_time() | - +---------------+-----------------+ - -- timeout parameter of cpu_percent* functions defaults to 0.0 instead of 0.1. -- long deprecated psutil.error module is gone; exception classes now live in - "psutil" namespace only. -- Process instances' "retcode" attribute returned by psutil.wait_procs() has - been renamed to "returncode" for consistency with subprocess.Popen. - - -1.2.1 - 2013-11-25 -================== - -**Bug fixes** - -- #348: [Windows XP] fixed "ImportError: DLL load failed" occurring on module - import. -- #425: [Solaris] crash on import due to failure at determining BOOT_TIME. -- #443: [Linux] can't set CPU affinity on systems with more than 64 cores. - - -1.2.0 - 2013-11-20 -================== - -**Enhancements** - -- #439: assume os.getpid() if no argument is passed to psutil.Process - constructor. -- #440: new psutil.wait_procs() utility function which waits for multiple - processes to terminate. - -**Bug fixes** - -- #348: [Windows XP/Vista] fix "ImportError: DLL load failed" occurring on - module import. - - -1.1.3 - 2013-11-07 -================== - -**Bug fixes** - -- #442: [Linux] psutil won't compile on certain version of Linux because of - missing prlimit(2) syscall. - - -1.1.2 - 2013-10-22 -================== - -**Bug fixes** - -- #442: [Linux] psutil won't compile on Debian 6.0 because of missing - prlimit(2) syscall. - - -1.1.1 - 2013-10-08 -================== - -**Bug fixes** - -- #442: [Linux] psutil won't compile on kernels < 2.6.36 due to missing - prlimit(2) syscall. - - -1.1.0 - 2013-09-28 -================== - -**Enhancements** - -- #410: host tar.gz and windows binary files are on PYPI. -- #412: [Linux] get/set process resource limits. -- #415: [Windows] Process.get_children() is an order of magnitude faster. -- #426: [Windows] Process.name is an order of magnitude faster. -- #431: [UNIX] Process.name is slightly faster because it unnecessarily - retrieved also process cmdline. - -**Bug fixes** - -- #391: [Windows] psutil.cpu_times_percent() returns negative percentages. -- #408: STATUS_* and CONN_* constants don't properly serialize on JSON. -- #411: [Windows] examples/disk_usage.py may pop-up a GUI error. -- #413: [Windows] Process.get_memory_info() leaks memory. -- #414: [Windows] Process.exe on Windows XP may raise ERROR_INVALID_PARAMETER. -- #416: psutil.disk_usage() doesn't work well with unicode path names. -- #430: [Linux] process IO counters report wrong number of r/w syscalls. -- #435: [Linux] psutil.net_io_counters() might report erreneous NIC names. -- #436: [Linux] psutil.net_io_counters() reports a wrong 'dropin' value. - -**API changes** - -- #408: turn STATUS_* and CONN_* constants into plain Python strings. - - -1.0.1 - 2013-07-12 -================== - -**Bug fixes** - -- #405: network_io_counters(pernic=True) no longer works as intended in 1.0.0. - - -1.0.0 - 2013-07-10 -================== - -**Enhancements** - -- #18: Solaris support (yay!) (thanks Justin Venus) -- #367: Process.get_connections() 'status' strings are now constants. -- #380: test suite exits with non-zero on failure. (patch by floppymaster) -- #391: introduce unittest2 facilities and provide workarounds if unittest2 - is not installed (python < 2.7). - -**Bug fixes** - -- #374: [Windows] negative memory usage reported if process uses a lot of - memory. -- #379: [Linux] Process.get_memory_maps() may raise ValueError. -- #394: [OSX] Mapped memory regions report incorrect file name. -- #404: [Linux] sched_*affinity() are implicitly declared. (patch by Arfrever) - -**API changes** - -- Process.get_connections() 'status' field is no longer a string but a - constant object (psutil.CONN_*). -- Process.get_connections() 'local_address' and 'remote_address' fields - renamed to 'laddr' and 'raddr'. -- psutil.network_io_counters() renamed to psutil.net_io_counters(). - - -0.7.1 - 2013-05-03 -================== - -**Bug fixes** - -- #325: [BSD] psutil.virtual_memory() can raise SystemError. - (patch by Jan Beich) -- #370: [BSD] Process.get_connections() requires root. (patch by John Baldwin) -- #372: [BSD] different process methods raise NoSuchProcess instead of - AccessDenied. - - -0.7.0 - 2013-04-12 -================== - -**Enhancements** - -- #233: code migrated to Mercurial (yay!) -- #246: psutil.error module is deprecated and scheduled for removal. -- #328: [Windows] process IO nice/priority support. -- #359: psutil.get_boot_time() -- #361: [Linux] psutil.cpu_times() now includes new 'steal', 'guest' and - 'guest_nice' fields available on recent Linux kernels. - Also, psutil.cpu_percent() is more accurate. -- #362: cpu_times_percent() (per-CPU-time utilization as a percentage) - -**Bug fixes** - -- #234: [Windows] disk_io_counters() fails to list certain disks. -- #264: [Windows] use of psutil.disk_partitions() may cause a message box to - appear. -- #313: [Linux] psutil.virtual_memory() and psutil.swap_memory() can crash on - certain exotic Linux flavors having an incomplete /proc interface. - If that's the case we now set the unretrievable stats to 0 and raise a - RuntimeWarning. -- #315: [OSX] fix some compilation warnings. -- #317: [Windows] cannot set process CPU affinity above 31 cores. -- #319: [Linux] process get_memory_maps() raises KeyError 'Anonymous' on Debian - squeeze. -- #321: [UNIX] Process.ppid property is no longer cached as the kernel may set - the ppid to 1 in case of a zombie process. -- #323: [OSX] disk_io_counters()'s read_time and write_time parameters were - reporting microseconds not milliseconds. (patch by Gregory Szorc) -- #331: Process cmdline is no longer cached after first acces as it may change. -- #333: [OSX] Leak of Mach ports on OS X (patch by rsesek@google.com) -- #337: [Linux] process methods not working because of a poor /proc - implementation will raise NotImplementedError rather than RuntimeError - and Process.as_dict() will not blow up. (patch by Curtin1060) -- #338: [Linux] disk_io_counters() fails to find some disks. -- #339: [FreeBSD] get_pid_list() can allocate all the memory on system. -- #341: [Linux] psutil might crash on import due to error in retrieving system - terminals map. -- #344: [FreeBSD] swap_memory() might return incorrect results due to - kvm_open(3) not being called. (patch by Jean Sebastien) -- #338: [Linux] disk_io_counters() fails to find some disks. -- #351: [Windows] if psutil is compiled with mingw32 (provided installers for - py2.4 and py2.5 are) disk_io_counters() will fail. (Patch by m.malycha) -- #353: [OSX] get_users() returns an empty list on OSX 10.8. -- #356: Process.parent now checks whether parent PID has been reused in which - case returns None. -- #365: Process.set_nice() should check PID has not been reused by another - process. -- #366: [FreeBSD] get_memory_maps(), get_num_fds(), get_open_files() and - getcwd() Process methods raise RuntimeError instead of AccessDenied. - -**API changes** - -- Process.cmdline property is no longer cached after first access. -- Process.ppid property is no longer cached after first access. -- [Linux] Process methods not working because of a poor /proc implementation - will raise NotImplementedError instead of RuntimeError. -- psutil.error module is deprecated and scheduled for removal. - - -0.6.1 - 2012-08-16 -================== - -**Enhancements** - -- #316: process cmdline property now makes a better job at guessing the process - executable from the cmdline. - -**Bug fixes** - -- #316: process exe was resolved in case it was a symlink. -- #318: python 2.4 compatibility was broken. - -**API changes** - -- process exe can now return an empty string instead of raising AccessDenied. -- process exe is no longer resolved in case it's a symlink. - - -0.6.0 - 2012-08-13 -================== - -**Enhancements** - -- #216: [POSIX] get_connections() UNIX sockets support. -- #220: [FreeBSD] get_connections() has been rewritten in C and no longer - requires lsof. -- #222: [OSX] add support for process cwd. -- #261: process extended memory info. -- #295: [OSX] process executable path is now determined by asking the OS - instead of being guessed from process cmdline. -- #297: [OSX] the Process methods below were always raising AccessDenied for - any process except the current one. Now this is no longer true. Also - they are 2.5x faster. - - name - - get_memory_info() - - get_memory_percent() - - get_cpu_times() - - get_cpu_percent() - - get_num_threads() -- #300: examples/pmap.py script. -- #301: process_iter() now yields processes sorted by their PIDs. -- #302: process number of voluntary and involuntary context switches. -- #303: [Windows] the Process methods below were always raising AccessDenied - for any process not owned by current user. Now this is no longer true: - - create_time - - get_cpu_times() - - get_cpu_percent() - - get_memory_info() - - get_memory_percent() - - get_num_handles() - - get_io_counters() -- #305: add examples/netstat.py script. -- #311: system memory functions has been refactorized and rewritten and now - provide a more detailed and consistent representation of the system - memory. New psutil.virtual_memory() function provides the following - memory amounts: - - total - - available - - percent - - used - - active [POSIX] - - inactive [POSIX] - - buffers (BSD, Linux) - - cached (BSD, OSX) - - wired (OSX, BSD) - - shared [FreeBSD] - New psutil.swap_memory() provides: - - total - - used - - free - - percent - - sin (no. of bytes the system has swapped in from disk (cumulative)) - - sout (no. of bytes the system has swapped out from disk (cumulative)) - All old memory-related functions are deprecated. - Also two new example scripts were added: free.py and meminfo.py. -- #312: psutil.network_io_counters() namedtuple includes 4 new fields: - errin, errout dropin and dropout, reflecting the number of packets - dropped and with errors. - -**Bugfixes** - -- #298: [OSX and BSD] memory leak in get_num_fds(). -- #299: potential memory leak every time PyList_New(0) is used. -- #303: [Windows] potential heap corruption in get_num_threads() and - get_status() Process methods. -- #305: [FreeBSD] psutil can't compile on FreeBSD 9 due to removal of utmp.h. -- #306: at C level, errors are not checked when invoking Py* functions which - create or manipulate Python objects leading to potential memory related - errors and/or segmentation faults. -- #307: [FreeBSD] values returned by psutil.network_io_counters() are wrong. -- #308: [BSD / Windows] psutil.virtmem_usage() wasn't actually returning - information about swap memory usage as it was supposed to do. It does - now. -- #309: get_open_files() might not return files which can not be accessed - due to limited permissions. AccessDenied is now raised instead. - -**API changes** - -- psutil.phymem_usage() is deprecated (use psutil.virtual_memory()) -- psutil.virtmem_usage() is deprecated (use psutil.swap_memory()) -- psutil.phymem_buffers() on Linux is deprecated (use psutil.virtual_memory()) -- psutil.cached_phymem() on Linux is deprecated (use psutil.virtual_memory()) -- [Windows and BSD] psutil.virtmem_usage() now returns information about swap - memory instead of virtual memory. - - -0.5.1 - 2012-06-29 -================== - -**Enhancements** - -- #293: [Windows] process executable path is now determined by asking the OS - instead of being guessed from process cmdline. - -**Bugfixes** - -- #292: [Linux] race condition in process files/threads/connections. -- #294: [Windows] Process CPU affinity is only able to set CPU #0. - - -0.5.0 - 2012-06-27 -================== - -**Enhancements** - -- #195: [Windows] number of handles opened by process. -- #209: psutil.disk_partitions() now provides also mount options. -- #229: list users currently connected on the system (psutil.get_users()). -- #238: [Linux, Windows] process CPU affinity (get and set). -- #242: Process.get_children(recursive=True): return all process - descendants. -- #245: [POSIX] Process.wait() incrementally consumes less CPU cycles. -- #257: [Windows] removed Windows 2000 support. -- #258: [Linux] Process.get_memory_info() is now 0.5x faster. -- #260: process's mapped memory regions. (Windows patch by wj32.64, OSX patch - by Jeremy Whitlock) -- #262: [Windows] psutil.disk_partitions() was slow due to inspecting the - floppy disk drive also when "all" argument was False. -- #273: psutil.get_process_list() is deprecated. -- #274: psutil no longer requires 2to3 at installation time in order to work - with Python 3. -- #278: new Process.as_dict() method. -- #281: ppid, name, exe, cmdline and create_time properties of Process class - are now cached after being accessed. -- #282: psutil.STATUS_* constants can now be compared by using their string - representation. -- #283: speedup Process.is_running() by caching its return value in case the - process is terminated. -- #284: [POSIX] per-process number of opened file descriptors. -- #287: psutil.process_iter() now caches Process instances between calls. -- #290: Process.nice property is deprecated in favor of new get_nice() and - set_nice() methods. - -**Bugfixes** - -- #193: psutil.Popen constructor can throw an exception if the spawned process - terminates quickly. -- #240: [OSX] incorrect use of free() for Process.get_connections(). -- #244: [POSIX] Process.wait() can hog CPU resources if called against a - process which is not our children. -- #248: [Linux] psutil.network_io_counters() might return erroneous NIC names. -- #252: [Windows] process getcwd() erroneously raise NoSuchProcess for - processes owned by another user. It now raises AccessDenied instead. -- #266: [Windows] psutil.get_pid_list() only shows 1024 processes. - (patch by Amoser) -- #267: [OSX] Process.get_connections() - an erroneous remote address was - returned. (Patch by Amoser) -- #272: [Linux] Porcess.get_open_files() - potential race condition can lead to - unexpected NoSuchProcess exception. Also, we can get incorrect reports - of not absolutized path names. -- #275: [Linux] Process.get_io_counters() erroneously raise NoSuchProcess on - old Linux versions. Where not available it now raises - NotImplementedError. -- #286: Process.is_running() doesn't actually check whether PID has been - reused. -- #314: Process.get_children() can sometimes return non-children. - -**API changes** - -- Process.nice property is deprecated in favor of new get_nice() and set_nice() - methods. -- psutil.get_process_list() is deprecated. -- ppid, name, exe, cmdline and create_time properties of Process class are now - cached after being accessed, meaning NoSuchProcess will no longer be raised - in case the process is gone in the meantime. -- psutil.STATUS_* constants can now be compared by using their string - representation. - - -0.4.1 - 2011-12-14 -================== - -**Bugfixes** - -- #228: some example scripts were not working with python 3. -- #230: [Windows / OSX] memory leak in Process.get_connections(). -- #232: [Linux] psutil.phymem_usage() can report erroneous values which are - different than "free" command. -- #236: [Windows] memory/handle leak in Process's get_memory_info(), - suspend() and resume() methods. - - -0.4.0 - 2011-10-29 -================== - -**Enhancements** - -- #150: network I/O counters. (OSX and Windows patch by Jeremy Whitlock) -- #154: [FreeBSD] add support for process getcwd() -- #157: [Windows] provide installer for Python 3.2 64-bit. -- #198: Process.wait(timeout=0) can now be used to make wait() return - immediately. -- #206: disk I/O counters. (OSX and Windows patch by Jeremy Whitlock) -- #213: examples/iotop.py script. -- #217: Process.get_connections() now has a "kind" argument to filter - for connections with different criteria. -- #221: [FreeBSD] Process.get_open_files has been rewritten in C and no longer - relies on lsof. -- #223: examples/top.py script. -- #227: examples/nettop.py script. - -**Bugfixes** - -- #135: [OSX] psutil cannot create Process object. -- #144: [Linux] no longer support 0 special PID. -- #188: [Linux] psutil import error on Linux ARM architectures. -- #194: [POSIX] psutil.Process.get_cpu_percent() now reports a percentage over - 100 on multicore processors. -- #197: [Linux] Process.get_connections() is broken on platforms not - supporting IPv6. -- #200: [Linux] psutil.NUM_CPUS not working on armel and sparc architectures - and causing crash on module import. -- #201: [Linux] Process.get_connections() is broken on big-endian - architectures. -- #211: Process instance can unexpectedly raise NoSuchProcess if tested for - equality with another object. -- #218: [Linux] crash at import time on Debian 64-bit because of a missing - line in /proc/meminfo. -- #226: [FreeBSD] crash at import time on FreeBSD 7 and minor. - - -0.3.0 - 2011-07-08 -================== - -**Enhancements** - -- #125: system per-cpu percentage utilization and times. -- #163: per-process associated terminal (TTY). -- #171: added get_phymem() and get_virtmem() functions returning system - memory information (total, used, free) and memory percent usage. - total_* avail_* and used_* memory functions are deprecated. -- #172: disk usage statistics. -- #174: mounted disk partitions. -- #179: setuptools is now used in setup.py - -**Bugfixes** - -- #159: SetSeDebug() does not close handles or unset impersonation on return. -- #164: [Windows] wait function raises a TimeoutException when a process - returns -1 . -- #165: process.status raises an unhandled exception. -- #166: get_memory_info() leaks handles hogging system resources. -- #168: psutil.cpu_percent() returns erroneous results when used in - non-blocking mode. (patch by Philip Roberts) -- #178: OSX - Process.get_threads() leaks memory -- #180: [Windows] Process's get_num_threads() and get_threads() methods can - raise NoSuchProcess exception while process still exists. - - -0.2.1 - 2011-03-20 -================== - -**Enhancements** - -- #64: per-process I/O counters. -- #116: per-process wait() (wait for process to terminate and return its exit - code). -- #134: per-process get_threads() returning information (id, user and kernel - times) about threads opened by process. -- #136: process executable path on FreeBSD is now determined by asking the - kernel instead of guessing it from cmdline[0]. -- #137: per-process real, effective and saved user and group ids. -- #140: system boot time. -- #142: per-process get and set niceness (priority). -- #143: per-process status. -- #147: per-process I/O nice (priority) - Linux only. -- #148: psutil.Popen class which tidies up subprocess.Popen and psutil.Process - in a unique interface. -- #152: [OSX] get_process_open_files() implementation has been rewritten - in C and no longer relies on lsof resulting in a 3x speedup. -- #153: [OSX] get_process_connection() implementation has been rewritten - in C and no longer relies on lsof resulting in a 3x speedup. - -**Bugfixes** - -- #83: process cmdline is empty on OSX 64-bit. -- #130: a race condition can cause IOError exception be raised on - Linux if process disappears between open() and subsequent read() calls. -- #145: WindowsError was raised instead of psutil.AccessDenied when using - process resume() or suspend() on Windows. -- #146: 'exe' property on Linux can raise TypeError if path contains NULL - bytes. -- #151: exe and getcwd() for PID 0 on Linux return inconsistent data. - -**API changes** - -- Process "uid" and "gid" properties are deprecated in favor of "uids" and - "gids" properties. - - -0.2.0 - 2010-11-13 -================== - -**Enhancements** - -- #79: per-process open files. -- #88: total system physical cached memory. -- #88: total system physical memory buffers used by the kernel. -- #91: per-process send_signal() and terminate() methods. -- #95: NoSuchProcess and AccessDenied exception classes now provide "pid", - "name" and "msg" attributes. -- #97: per-process children. -- #98: Process.get_cpu_times() and Process.get_memory_info now return - a namedtuple instead of a tuple. -- #103: per-process opened TCP and UDP connections. -- #107: add support for Windows 64 bit. (patch by cjgohlke) -- #111: per-process executable name. -- #113: exception messages now include process name and pid. -- #114: process username Windows implementation has been rewritten in pure - C and no longer uses WMI resulting in a big speedup. Also, pywin32 is no - longer required as a third-party dependancy. (patch by wj32) -- #117: added support for Windows 2000. -- #123: psutil.cpu_percent() and psutil.Process.cpu_percent() accept a - new 'interval' parameter. -- #129: per-process number of threads. - -**Bugfixes** - -- #80: fixed warnings when installing psutil with easy_install. -- #81: psutil fails to compile with Visual Studio. -- #94: suspend() raises OSError instead of AccessDenied. -- #86: psutil didn't compile against FreeBSD 6.x. -- #102: orphaned process handles obtained by using OpenProcess in C were - left behind every time Process class was instantiated. -- #111: path and name Process properties report truncated or erroneous - values on UNIX. -- #120: cpu_percent() always returning 100% on OS X. -- #112: uid and gid properties don't change if process changes effective - user/group id at some point. -- #126: ppid, uid, gid, name, exe, cmdline and create_time properties are - no longer cached and correctly raise NoSuchProcess exception if the process - disappears. - -**API changes** - -- psutil.Process.path property is deprecated and works as an alias for "exe" - property. -- psutil.Process.kill(): signal argument was removed - to send a signal to the - process use send_signal(signal) method instead. -- psutil.Process.get_memory_info() returns a nametuple instead of a tuple. -- psutil.cpu_times() returns a nametuple instead of a tuple. -- New psutil.Process methods: get_open_files(), get_connections(), - send_signal() and terminate(). -- ppid, uid, gid, name, exe, cmdline and create_time properties are no longer - cached and raise NoSuchProcess exception if process disappears. -- psutil.cpu_percent() no longer returns immediately (see issue 123). -- psutil.Process.get_cpu_percent() and psutil.cpu_percent() no longer returns - immediately by default (see issue 123). - - -0.1.3 - 2010-03-02 -================== - -**Enhancements** - -- #14: per-process username -- #51: per-process current working directory (Windows and Linux only) -- #59: Process.is_running() is now 10 times faster -- #61: added supoprt for FreeBSD 64 bit -- #71: implemented suspend/resume process -- #75: python 3 support - -**Bugfixes** - -- #36: process cpu_times() and memory_info() functions succeeded also for dead - processes while a NoSuchProcess exception is supposed to be raised. -- #48: incorrect size for mib array defined in getcmdargs for BSD -- #49: possible memory leak due to missing free() on error condition on -- #50: fixed getcmdargs() memory fragmentation on BSD -- #55: test_pid_4 was failing on Windows Vista -- #57: some unit tests were failing on systems where no swap memory is - available -- #58: is_running() is now called before kill() to make sure we are going - to kill the correct process. -- #73: virtual memory size reported on OS X includes shared library size -- #77: NoSuchProcess wasn't raised on Process.create_time if kill() was - used first. - - -0.1.2 - 2009-05-06 -================== - -**Enhancements** - -- #32: Per-process CPU user/kernel times -- #33: Process create time -- #34: Per-process CPU utilization percentage -- #38: Per-process memory usage (bytes) -- #41: Per-process memory utilization (percent) -- #39: System uptime -- #43: Total system virtual memory -- #46: Total system physical memory -- #44: Total system used/free virtual and physical memory - -**Bugfixes** - -- #36: [Windows] NoSuchProcess not raised when accessing timing methods. -- #40: test_get_cpu_times() failing on FreeBSD and OS X. -- #42: [Windows] get_memory_percent() raises AccessDenied. - - -0.1.1 - 2009-03-06 -================== - -**Enhancements** - -- #4: FreeBSD support for all functions of psutil -- #9: Process.uid and Process.gid now retrieve process UID and GID. -- #11: Support for parent/ppid - Process.parent property returns a - Process object representing the parent process, and Process.ppid returns - the parent PID. -- #12 & 15: - NoSuchProcess exception now raised when creating an object - for a nonexistent process, or when retrieving information about a process - that has gone away. -- #21: AccessDenied exception created for raising access denied errors - from OSError or WindowsError on individual platforms. -- #26: psutil.process_iter() function to iterate over processes as - Process objects with a generator. -- #?: Process objects can now also be compared with == operator for equality - (PID, name, command line are compared). - -**Bugfixes** - -- #16: [Windows] Special case for "System Idle Process" (PID 0) which - otherwise would return an "invalid parameter" exception. -- #17: get_process_list() ignores NoSuchProcess and AccessDenied - exceptions during building of the list. -- #22: [Windows] Process(0).kill() was failing with an unset exception. -- #23: Special case for pid_exists(0) -- #24: [Windows] Process(0).kill() now raises AccessDenied exception instead - of WindowsError. -- #30: psutil.get_pid_list() was returning two ins +- https://psutil.io/changelog/ diff --git a/IDEAS b/IDEAS deleted file mode 100644 index 9eb7d76266..0000000000 --- a/IDEAS +++ /dev/null @@ -1,147 +0,0 @@ -TODO -==== - -A collection of ideas and notes about stuff to implement in future versions. -"#NNN" occurrences refer to bug tracker issues at: -https://github.com/giampaolo/psutil/issues - -PLATFORMS -========= - -- #355 (patch): Android -- #605 (branch): AIX -- #276: GNU/Hurd -- DragonFlyBSD -- HP-UX - -FEATURES -======== - -- (Linux): from /proc/pid/stat we can also retrieve process and children guest - times (time spent running a virtual CPU for a guest OS). - -- #809: (Linux) per-process resource limits. - -- (UNIX) process root (different from cwd) - -- #782: (UNIX) process num of signals received. - -- (Linux) locked files via /proc/locks: - https://www.centos.org/docs/5/html/5.2/Deployment_Guide/s2-proc-locks.html - -- #371: CPU temperature (apparently OSX and Linux only; on Linux it requires - lm-sensors lib). - -- #269: NIC rx/tx queue. This should probably go into net_if_stats(). - Figure out on what platforms this is supported: - Linux: yes - Others: ? - -- Process.threads(): thread names; patch for OSX available at: - https://code.google.com/p/plcrashreporter/issues/detail?id=65 - Sample code: - https://github.com/janmojzis/pstree/blob/master/proc_kvm.c - -- Asynchronous psutil.Popen (see http://bugs.python.org/issue1191964) - -- (Windows) fall back on using WMIC for Process methods returning AccessDenied - -- #613: thread names. - -- #604: emulate os.getloadavg() on Windows - -- scripts/taskmgr-gui.py (using tk). - -- system-wide number of open file descriptors: - - https://jira.hyperic.com/browse/SIGAR-30 - - http://www.netadmintools.com/part295.html - -- Number of system threads. - - Windows: http://msdn.microsoft.com/en-us/library/windows/desktop/ms684824(v=vs.85).aspx - -- #357: what CPU a process is on. - -- Doc / wiki which compares similarities between UNIX cli tools and psutil. - Example: - ``` - df -a -> psutil.disk_partitions - lsof -> psutil.Process.open_files() and psutil.Process.open_connections() - killall-> (actual script) - tty -> psutil.Process.terminal() - who -> psutil.users() - ``` - -- psutil.proc_tree() something which obtains a {pid:ppid, ...} dict for - all running processes in one shot. This can be factored out from - Process.children() and exposed as a first class function. - PROS: on Windows we can take advantage of _psutil_windows.ppid_map() - which is faster than iterating over all pids and calling ppid(). - CONS: scripts/pstree.py shows this can be easily done in the user code - so maybe it's not worth the addition. - -- advanced cmdline interface exposing the whole API and providing different - kind of outputs (e.g. pprinted, colorized, json). - -- [Linux]: process cgroups (http://en.wikipedia.org/wiki/Cgroups). They look - similar to prlimit() in terms of functionality but uglier (they should allow - limiting per-process network IO resources though, which is great). Needs - further reading. - -- Python 3.3. exposed different sched.h functions: - http://docs.python.org/dev/whatsnew/3.3.html#os - http://bugs.python.org/issue12655 - http://docs.python.org/dev/library/os.html#interface-to-the-scheduler - It might be worth to take a look and figure out whether we can include some - of those in psutil. - Also, we can probably reimplement wait_pid() on POSIX which is currently - implemented as a busy-loop. - -- os.times() provides 'elapsed' times (cpu_times() might). - -- ...also guest_time and cguest_time on Linux. - -- Enrich exception classes hierarchy on Python >= 3.3 / post PEP-3151 so that: - - NoSuchProcess inherits from ProcessLookupError - - AccessDenied inherits from PermissionError - - TimeoutExpired inherits from TimeoutError (debatable) - See: http://docs.python.org/3/library/exceptions.html#os-exceptions - -- Process.threads() might grow an extra "id" parameter so that it can be - used as such: - ``` - >>> p = psutil.Process(os.getpid()) - >>> p.threads(id=psutil.current_thread_id()) - thread(id=2539, user_time=0.03, system_time=0.02) - >>> - ``` - Note: this leads to questions such as "should we have a custom NoSuchThread - exception? Also see issue #418. - Note #2: this would work with os.getpid() only. - psutil.current_thread_id() might be desirable as per issue #418 though. - -- should psutil.TimeoutExpired exception have a 'msg' kwarg similar to - NoSuchProcess and AccessDenied? Not that we need it, but currently we - cannot raise a TimeoutExpired exception with a specific error string. - -- process_iter() might grow an "attrs" parameter similar to Process.as_dict() - invoke the necessary methods and include the results into a "cache" - attribute attached to the returned Process instances so that one can avoid - catching NSP and AccessDenied: - for p in process_iter(attrs=['cpu_percent']): - print(p.cache['cpu_percent']) - This also leads questions as whether we should introduce a sorting order. - -- round Process.memory_percent() result? - -- #550: number of threads per core. - -- Have psutil.Process().cpu_affinity([]) be an alias for "all CPUs"? - - -RESOURCES -========= - -- sigar: https://github.com/hyperic/sigar (Java) -- zabbix: https://zabbix.org/wiki/Get_Zabbix -- libstatgrab: http://www.i-scream.org/libstatgrab/ -- top: http://www.unixtop.org/ diff --git a/INSTALL.rst b/INSTALL.rst index 05bbc9c354..3165300fe4 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -1,120 +1,3 @@ -*Note: pip is the easiest way to install psutil. -It is shipped by default with Python 2.7.9+ and 3.4+. If you're using an -older Python version* `install pip `__ -*first.* If you cloned psutil source code you can also install it with -``make install-pip``. +Installation instructions have moved to: -Permission issues -================= - -Except for Linux, the commands below assume you're running as root. -If you're not and you bump into permission errors you can either: - -* prepend ``sudo``, e.g.: - -:: - - sudo pip install psutil - -* install psutil for your user only (not at system level): - -:: - - pip install --user psutil - -Linux -===== - -Ubuntu / Debian (use ``python3-dev`` and ``python3-pip`` for python 3):: - - sudo apt-get install gcc python-dev python-pip - pip install psutil - -RedHat (use ``python3-devel`` and ``python3-pip`` for python 3):: - - sudo yum install gcc python-devel python-pip - pip install psutil - -OSX -=== - -Install `XcodeTools `__ -first, then: - -:: - - pip install psutil - -Windows -======= - -The easiest way to install psutil on Windows is to just use the pre-compiled -exe/wheel installers on -`PYPI `__ via pip:: - - C:\Python27\python.exe -m pip install psutil - -If you want to compile psutil from sources you'll need **Visual Studio** -(Mingw32 is no longer supported): - -* Python 2.6, 2.7: `VS-2008 `__ -* Python 3.3, 3.4: `VS-2010 `__ -* Python 3.5+: `VS-2015 `__ - -Compiling 64 bit versions of Python 2.6 and 2.7 with VS 2008 requires -`Windows SDK and .NET Framework 3.5 SP1 `__. -Once installed run vcvars64.bat, then you can finally compile (see -`here `__). -To compile / install psutil from sources on Windows run:: - - make.bat build - make.bat install - -FreeBSD -======= - -:: - - pkg install python gcc - python -m pip install psutil - -OpenBSD -======= - -:: - - export PKG_PATH=http://ftp.usa.openbsd.org/pub/OpenBSD/`uname -r`/packages/`arch -s` - pkg_add -v python gcc - python -m pip install psutil - -NetBSD -====== - -:: - - export PKG_PATH="ftp.netbsd.org/pub/pkgsrc/packages/NetBSD/`uname -m`/`uname -r`/All" - pkg_add -v pkgin - pkgin install python gcc - python -m pip install psutil - -Solaris -======= - -If ``cc`` compiler is not installed create a symlink to ``gcc``: - -:: - - sudo ln -s /usr/bin/gcc /usr/local/bin/cc - -Install: - -:: - - pkg install gcc - python -m pip install psutil - -Dev Guide -========= - -If you plan on hacking on psutil you may want to take a look at the -`dev guide `__. +- https://psutil.io/install/ diff --git a/LICENSE b/LICENSE index e91b1359a2..cff5eb74e1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ -psutil is distributed under BSD license reproduced below. +BSD 3-Clause License -Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola' +Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola All rights reserved. Redistribution and use in source and binary forms, with or without modification, @@ -8,9 +8,11 @@ are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of the psutil authors nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/MANIFEST.in b/MANIFEST.in index 67280314b9..40739f4dd9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,22 +1,204 @@ -include .coveragerc -include .git-pre-commit +include .clang-format +include .dprint.jsonc include .gitignore -include .travis.yml -include appveyor.yml -include CREDITS -include DEVGUIDE.rst +include CONTRIBUTING.md include HISTORY.rst -include IDEAS include INSTALL.rst include LICENSE -include make.bat -include Makefile include MANIFEST.in +include Makefile include README.rst +include SECURITY.md +include _bootstrap.py +include psutil/__init__.py +include psutil/_common.py +include psutil/_enums.py +include psutil/_ntuples.py +include psutil/_psaix.py +include psutil/_psbsd.py +include psutil/_pslinux.py +include psutil/_psosx.py +include psutil/_psposix.py +include psutil/_pssunos.py +include psutil/_psutil_aix.c +include psutil/_psutil_bsd.c +include psutil/_psutil_linux.c +include psutil/_psutil_osx.c +include psutil/_psutil_sunos.c +include psutil/_psutil_windows.c +include psutil/_pswindows.py +include psutil/arch/aix/cpu.c +include psutil/arch/aix/disk.c +include psutil/arch/aix/ifaddrs.c +include psutil/arch/aix/ifaddrs.h +include psutil/arch/aix/init.h +include psutil/arch/aix/mem.c +include psutil/arch/aix/net.c +include psutil/arch/aix/net_kernel_structs.h +include psutil/arch/aix/proc.c +include psutil/arch/aix/socks.c +include psutil/arch/aix/sys.c +include psutil/arch/all/errors.c +include psutil/arch/all/init.c +include psutil/arch/all/init.h +include psutil/arch/all/pids.c +include psutil/arch/all/str.c +include psutil/arch/all/utils.c +include psutil/arch/bsd/cpu.c +include psutil/arch/bsd/disk.c +include psutil/arch/bsd/heap.c +include psutil/arch/bsd/init.c +include psutil/arch/bsd/init.h +include psutil/arch/bsd/mem.c +include psutil/arch/bsd/net.c +include psutil/arch/bsd/proc.c +include psutil/arch/bsd/proc_utils.c +include psutil/arch/bsd/sys.c +include psutil/arch/freebsd/cpu.c +include psutil/arch/freebsd/disk.c +include psutil/arch/freebsd/init.h +include psutil/arch/freebsd/mem.c +include psutil/arch/freebsd/pids.c +include psutil/arch/freebsd/proc.c +include psutil/arch/freebsd/proc_socks.c +include psutil/arch/freebsd/sensors.c +include psutil/arch/freebsd/sys_socks.c +include psutil/arch/linux/disk.c +include psutil/arch/linux/heap.c +include psutil/arch/linux/init.h +include psutil/arch/linux/mem.c +include psutil/arch/linux/net.c +include psutil/arch/linux/proc.c +include psutil/arch/netbsd/cpu.c +include psutil/arch/netbsd/disk.c +include psutil/arch/netbsd/init.h +include psutil/arch/netbsd/mem.c +include psutil/arch/netbsd/pids.c +include psutil/arch/netbsd/proc.c +include psutil/arch/netbsd/socks.c +include psutil/arch/openbsd/cpu.c +include psutil/arch/openbsd/disk.c +include psutil/arch/openbsd/init.h +include psutil/arch/openbsd/mem.c +include psutil/arch/openbsd/pids.c +include psutil/arch/openbsd/proc.c +include psutil/arch/openbsd/socks.c +include psutil/arch/openbsd/users.c +include psutil/arch/osx/cpu.c +include psutil/arch/osx/disk.c +include psutil/arch/osx/heap.c +include psutil/arch/osx/init.c +include psutil/arch/osx/init.h +include psutil/arch/osx/mem.c +include psutil/arch/osx/net.c +include psutil/arch/osx/pids.c +include psutil/arch/osx/proc.c +include psutil/arch/osx/proc_utils.c +include psutil/arch/osx/sensors.c +include psutil/arch/osx/sys.c +include psutil/arch/posix/init.c +include psutil/arch/posix/init.h +include psutil/arch/posix/net.c +include psutil/arch/posix/pids.c +include psutil/arch/posix/proc.c +include psutil/arch/posix/sysctl.c +include psutil/arch/posix/users.c +include psutil/arch/sunos/cpu.c +include psutil/arch/sunos/disk.c +include psutil/arch/sunos/environ.c +include psutil/arch/sunos/init.h +include psutil/arch/sunos/mem.c +include psutil/arch/sunos/net.c +include psutil/arch/sunos/proc.c +include psutil/arch/sunos/sys.c +include psutil/arch/windows/cpu.c +include psutil/arch/windows/disk.c +include psutil/arch/windows/heap.c +include psutil/arch/windows/init.c +include psutil/arch/windows/init.h +include psutil/arch/windows/mem.c +include psutil/arch/windows/net.c +include psutil/arch/windows/ntextapi.h +include psutil/arch/windows/pids.c +include psutil/arch/windows/proc.c +include psutil/arch/windows/proc_handles.c +include psutil/arch/windows/proc_peb.c +include psutil/arch/windows/proc_utils.c +include psutil/arch/windows/security.c +include psutil/arch/windows/sensors.c +include psutil/arch/windows/services.c +include psutil/arch/windows/socks.c +include psutil/arch/windows/sys.c +include psutil/arch/windows/wmi.c +include pyproject.toml +include scripts/battery.py +include scripts/cpu_distribution.py +include scripts/disk_usage.py +include scripts/fans.py +include scripts/free.py +include scripts/ifconfig.py +include scripts/internal/README +include scripts/internal/bench_oneshot.py +include scripts/internal/bench_oneshot_2.py +include scripts/internal/cfarm-test.sh +include scripts/internal/convert_readme.py +include scripts/internal/docs/build_versions.py +include scripts/internal/docs/find_adopters.py +include scripts/internal/docs/new_blog_post.py +include scripts/internal/docs/refresh_adoption_stats.py +include scripts/internal/docs/rst_unused_targets.py +include scripts/internal/download_wheels.py +include scripts/internal/find_broken_links.py +include scripts/internal/generate_manifest.py +include scripts/internal/git_pre_commit.py +include scripts/internal/install-pydeps.sh +include scripts/internal/install-sysdeps.sh +include scripts/internal/install_pip.py +include scripts/internal/print_access_denied.py +include scripts/internal/print_announce.py +include scripts/internal/print_api_speed.py +include scripts/internal/print_dist.py +include scripts/internal/print_downloads.py +include scripts/internal/print_hashes.py +include scripts/internal/print_sysinfo.py +include scripts/internal/purge_installation.py +include scripts/iotop.py +include scripts/killall.py +include scripts/meminfo.py +include scripts/netstat.py +include scripts/nettop.py +include scripts/pidof.py +include scripts/pmap.py +include scripts/procinfo.py +include scripts/procsmem.py +include scripts/ps.py +include scripts/pstree.py +include scripts/sensors.py +include scripts/temperatures.py +include scripts/top.py +include scripts/who.py +include scripts/winservices.py include setup.py -include tox.ini -recursive-exclude docs/_build * -recursive-include .ci * -recursive-include docs * -recursive-include psutil *.py *.c *.h README* -recursive-include scripts *.py +include tests/README.md +include tests/__init__.py +include tests/conftest.py +include tests/test_aix.py +include tests/test_bsd.py +include tests/test_connections.py +include tests/test_contracts.py +include tests/test_heap.py +include tests/test_linux.py +include tests/test_memleaks.py +include tests/test_misc.py +include tests/test_osx.py +include tests/test_posix.py +include tests/test_process.py +include tests/test_process_all.py +include tests/test_scripts.py +include tests/test_sudo.py +include tests/test_sunos.py +include tests/test_system.py +include tests/test_testutils.py +include tests/test_type_hints.py +include tests/test_unicode.py +include tests/test_windows.py diff --git a/Makefile b/Makefile index b23249cd0d..87a16ce10b 100644 --- a/Makefile +++ b/Makefile @@ -1,129 +1,191 @@ -# Shortcuts for various tasks (UNIX only). -# To use a specific Python version run: "make install PYTHON=python3.3" - -# You can set these variables from the command line. -PYTHON = python -TSCRIPT = psutil/tests/runner.py - -# For internal use. -DEPS = coverage \ - flake8 \ - futures \ - ipdb \ - mock==1.0.1 \ - nose \ - pep8 \ - pyflakes \ - requests \ - sphinx \ - sphinx-pypi-upload \ - unittest2 - -all: test +# Shortcuts for various development tasks. +# +# - To use this on Windows install Git For Windows first, then launch a Git +# Bash Shell. +# - To use a specific Python version run: `make install PYTHON=python3.13`. +# - To append an argument to a command use ARGS, e.g: `make test ARGS="-k +# some_test`. +# - Needs GNU make >= 4.0, and must also parse with BSD make, which runs +# `make ci-test` on the *BSD CI. BSD make dies on a `:` inside `$(...)`, +# so keep those out of file scope. Other GNU-only bits are fine as long +# as they stay in targets the BSDs don't run. + +# Configurable +PYTHON = python3 +ARGS = +FILES = + +PYTHON_ENV_VARS = PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 PSUTIL_TESTING=1 PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 +SUDO = $(if $(filter $(OS),Windows_NT),,sudo -E) +DPRINT = ~/.dprint/bin/dprint +INSTALL_PYDEPS = PYTHON=$(PYTHON) ./scripts/internal/install-pydeps.sh + +# `make` called with no args is like `make help` +.DEFAULT_GOAL := help +.PHONY: build test + +# install git hook (skipped in worktrees, where .git is a file) +_ := $(shell test -d .git && mkdir -p .git/hooks/ && ln -sf ../../scripts/internal/git_pre_commit.py .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit) # =================================================================== # Install # =================================================================== -clean: - rm -f `find . -type f -name \*.py[co]` - rm -f `find . -type f -name \*.so` - rm -f `find . -type f -name \*.~` - rm -f `find . -type f -name \*.orig` - rm -f `find . -type f -name \*.bak` - rm -f `find . -type f -name \*.rej` - rm -rf `find . -type d -name __pycache__` - rm -rf *.core - rm -rf *.egg-info - rm -rf *\$testfile* - rm -rf .coverage - rm -rf .tox - rm -rf build/ - rm -rf dist/ - rm -rf docs/_build/ - rm -rf htmlcov/ - rm -rf tmp/ - -build: clean - $(PYTHON) setup.py build - @# copies *.so files in ./psutil directory in order to allow - @# "import psutil" when using the interactive interpreter from within - @# this directory. - $(PYTHON) setup.py build_ext -i - rm -rf tmp - -install: build - $(PYTHON) setup.py develop --user - rm -rf tmp - -uninstall: - cd ..; $(PYTHON) -m pip uninstall -y -v psutil - -install-pip: - # Install PIP (only if necessary). - $(PYTHON) -c "import sys, ssl, os, pkgutil, tempfile, atexit; \ - sys.exit(0) if pkgutil.find_loader('pip') else None; \ - pyexc = 'from urllib.request import urlopen' if sys.version_info[0] == 3 else 'from urllib2 import urlopen'; \ - exec(pyexc); \ - context = ssl._create_unverified_context() if hasattr(ssl, '_create_unverified_context') else None; \ - kw = dict(context=context) if context else {}; \ - req = urlopen('https://bootstrap.pypa.io/get-pip.py', **kw); \ - data = req.read(); \ - f = tempfile.NamedTemporaryFile(suffix='.py'); \ - atexit.register(f.close); \ - f.write(data); \ - f.flush(); \ - print('downloaded %s' % f.name); \ - code = os.system('%s %s --user' % (sys.executable, f.name)); \ - sys.exit(code);" - -# Install useful deps which are nice to have while developing / testing. -setup-dev-env: install-git-hooks install-pip - $(PYTHON) -m pip install --user --upgrade pip - $(PYTHON) -m pip install --user --upgrade $(DEPS) +clean: ## Remove all build files. + @rm -rfv `find . \ + -type d -name __pycache__ \ + -o -type f -name \*.bak \ + -o -type f -name \*.orig \ + -o -type f -name \*.pyc \ + -o -type f -name \*.pyd \ + -o -type f -name \*.pyo \ + -o -type f -name \*.rej \ + -o -type f -name \*.so \ + -o -type f -name \*.~ \ + -o -name \*@psutil-\*` + @rm -rfv \ + *.core \ + *.egg-info \ + *\@psutil-* \ + .coverage \ + .failed-tests.txt \ + .pytest_cache \ + .ruff_cache/ \ + .tests \ + build/ \ + dist/ \ + docs/_build/ \ + htmlcov/ \ + pytest-cache-files* \ + wheelhouse + +build: ## Compile (in parallel) without installing. + @# "build_ext -i" copies compiled *.so files in ./psutil directory in order + @# to allow "import psutil" when using the interactive interpreter from + @# within this directory. + $(PYTHON_ENV_VARS) $(PYTHON) setup.py build_ext --inplace + $(PYTHON_ENV_VARS) $(PYTHON) -c "import psutil" # make sure it actually worked + +install: ## Install this package as current user in edit / development mode. + # --no-build-isolation: reuse setuptools installed above instead of + # downloading another copy into a temporary build env. + $(PYTHON_ENV_VARS) $(INSTALL_PYDEPS) --no-build-isolation --editable . + $(PYTHON_ENV_VARS) $(PYTHON) -c "import psutil" # make sure it actually worked + +uninstall: ## Uninstall this package via pip. + cd ..; $(PYTHON_ENV_VARS) $(PYTHON) -m pip uninstall -y -v psutil || true + $(PYTHON_ENV_VARS) $(PYTHON) scripts/internal/purge_installation.py + +install-sysdeps: ## Install system deps needed to compile psutil. + ./scripts/internal/install-sysdeps.sh + +install-sysdeps-test: ## Install CLI tools needed to run unit tests. + ./scripts/internal/install-sysdeps.sh --test-only + +# --- + +install-pydeps-build: ## Install python deps necessary to compile psutil. + $(INSTALL_PYDEPS) --group build + +install-pydeps-test: ## Install python deps necessary to run unit tests. + $(INSTALL_PYDEPS) --group test + +install-pydeps-lint: ## Install python deps necessary to run linters. + $(INSTALL_PYDEPS) --group lint + +install-pydeps-docs: ## Install python deps necessary to build the doc. + $(INSTALL_PYDEPS) --group docs + +install-pydeps-dev: ## Install python deps meant for local development. + $(INSTALL_PYDEPS) --group dev # =================================================================== # Tests # =================================================================== -# Run all tests. -test: install - $(PYTHON) $(TSCRIPT) +# - cache dir on Windows often causes "Permission denied" errors +# - drop instafail on CI: conftest.py already repeats failures at the end, +# and instafail is only useful while developing locally +_PYTEST_EXTRA = `{ if [ "$$OS" = "Windows_NT" ]; then printf '%s ' '-o cache_dir=/tmp/pytest-psutil-cache'; fi; if [ -n "$$CI" ]; then printf '%s ' '-p no:instafail'; fi; }` -# Test psutil process-related APIs. -test-process: install - $(PYTHON) -m unittest -v psutil.tests.test_process +RUN_TEST = $(PYTHON_ENV_VARS) $(PYTHON) -m pytest --durations=5 $(_PYTEST_EXTRA) +RUN_TEST_MEMLEAKS = PYTHONMALLOC=malloc $(RUN_TEST) -k test_memleaks.py -# Test psutil system-related APIs. -test-system: install - $(PYTHON) -m unittest -v psutil.tests.test_system +# --- main -# Test misc. -test-misc: install - $(PYTHON) psutil/tests/test_misc.py +test: ## Run all tests (except memleak tests). + # To run a specific test do `make test ARGS=tests/test_process.py::TestProcess::test_cmdline` + $(RUN_TEST) $(ARGS) -# Test memory leaks. -test-memleaks: install - $(PYTHON) psutil/tests/test_memory_leaks.py +test-parallel: ## Run all tests (except memleak tests) in parallel. + $(RUN_TEST) -n auto --dist loadgroup -m 'not isolated' $(ARGS) + $(RUN_TEST) -m isolated $(ARGS) -# Run specific platform tests only. -test-platform: install - $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py +test-memleaks: ## Run memory leak tests. + $(RUN_TEST_MEMLEAKS) $(ARGS) -# Run a specific test by name; e.g. "make test-by-name disk_" will run -# all test methods containing "disk_" in their name. -# Requires "pip install nose". -test-by-name: install - @$(PYTHON) -m nose psutil/tests/*.py --nocapture -v -m $(filter-out $@,$(MAKECMDGOALS)) +test-memleaks-parallel: ## Run memory leak tests in parallel. + $(RUN_TEST_MEMLEAKS) -n auto $(ARGS) -# Same as above but for test_memory_leaks.py script. -test-memleaks-by-name: install - @$(PYTHON) -m nose test/test_memory_leaks.py --nocapture -v -m $(filter-out $@,$(MAKECMDGOALS)) +# --- individual -coverage: install - # Note: coverage options are controlled by .coveragerc file +test-process: ## Run process-related tests. + $(RUN_TEST) -k "test_process.py or test_proc or test_pid or Process or pids or pid_exists" $(ARGS) + +test-process-all: ## Run tests which iterate over all process PIDs. + $(RUN_TEST) -k test_process_all.py $(ARGS) + +test-system: ## Run system-related API tests. + $(RUN_TEST) -k "test_system.py or test_sys or System or disk or sensors or net_io_counters or net_if_addrs or net_if_stats or users or pids or win_service_ or boot_time" $(ARGS) + +test-misc: ## Run miscellaneous tests. + $(RUN_TEST) -k "test_misc.py or Misc" $(ARGS) + +test-scripts: ## Run scripts tests. + $(RUN_TEST) -k test_scripts.py $(ARGS) + +test-testutils: ## Run test utils tests. + $(RUN_TEST) -k test_testutils.py $(ARGS) + +test-unicode: ## Test APIs dealing with strings. + $(RUN_TEST) -k test_unicode.py $(ARGS) + +test-contracts: ## APIs sanity tests. + $(RUN_TEST) -k test_contracts.py $(ARGS) + +test-docs: ## Run doc sanity tests (outside testpaths, run on demand). + $(MAKE) -C docs test ARGS="$(ARGS)" + +test-bots: ## Run GitHub bot tests (outside testpaths, run on demand). + $(PYTHON) -m pytest -o addopts="" .github/workflows/tests/ $(ARGS) + +test-type-hints: ## Test type hints + $(RUN_TEST) -k test_type_hints.py $(ARGS) + +test-connections: ## Test psutil.net_connections() and Process.net_connections(). + $(RUN_TEST) -k "test_connections.py or net_" $(ARGS) + +test-heap: ## Test psutil.heap_*() APIs. + $(RUN_TEST) -k "test_heap.py or heap_" $(ARGS) + +test-posix: ## POSIX specific tests. + $(RUN_TEST) -k "test_posix.py or posix_ or Posix" $(ARGS) + +test-platform: ## Run specific platform tests only. + $(RUN_TEST) -k test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py $(ARGS) + +# --- special + +test-sudo: ## Run tests requiring root privileges. + # Use unittest runner because pytest may not be installed as root. + $(SUDO) $(PYTHON_ENV_VARS) $(PYTHON) -m unittest -v tests.test_sudo + +test-last-failed: ## Re-run tests which failed on last run + $(RUN_TEST) --last-failed $(ARGS) + +coverage: ## Run test coverage. rm -rf .coverage htmlcov - $(PYTHON) -m coverage run $(TSCRIPT) + $(PYTHON_ENV_VARS) $(PYTHON) -m coverage run -m pytest $(ARGS) $(PYTHON) -m coverage report @echo "writing results to htmlcov/index.html" $(PYTHON) -m coverage html @@ -133,48 +195,231 @@ coverage: install # Linters # =================================================================== -pep8: - @git ls-files | grep \\.py$ | xargs $(PYTHON) -m pep8 +# Return a shell pipeline that outputs one file per line. Uses +# $(FILES) if set, else "git ls-files" with given pattern(s). +_ls = $(if $(FILES), printf '%s\n' $(FILES), git ls-files $(1)) + +ruff: ## Run ruff linter. + @$(call _ls,'*.py') | xargs $(PYTHON) -m ruff check --output-format=concise + +black: ## Run black formatter. + @$(call _ls,'*.py') | xargs $(PYTHON) -m black --check --safe + +lint-c: ## Run C linter. + @$(call _ls,'*.c' '*.h') | xargs -P0 -I{} clang-format --dry-run --Werror {} -pyflakes: - @export PYFLAKES_NODOCTEST=1 && \ - git ls-files | grep \\.py$ | xargs $(PYTHON) -m pyflakes +dprint: ## Run linter for .md / .json / .yml / .js / .css files. + @$(DPRINT) check -flake8: - @git ls-files | grep \\.py$ | xargs $(PYTHON) -m flake8 +lint-rst: ## Run linter for .rst files. + @$(call _ls,'*.rst') | xargs $(PYTHON) scripts/internal/docs/rst_unused_targets.py + @$(call _ls,'*.rst') | xargs sphinx-lint --enable all --disable line-too-long + @$(call _ls,'*.rst') | xargs rstwrap --check + +lint-toml: ## Run linter for pyproject.toml. + @$(call _ls,'*.toml') | xargs toml-sort --check + +lint-all: ## Run all linters in parallel + $(MAKE) -j \ + black \ + ruff \ + lint-c \ + dprint \ + lint-rst \ + lint-toml + +# --- not mandatory linters (just run from time to time) + +pylint: ## Python pylint + @$(call _ls,'*.py') | xargs $(PYTHON) -m pylint --rcfile=pyproject.toml --jobs=0 $(ARGS) + +vulture: ## Find unused code + @$(call _ls,'*.py') | xargs $(PYTHON) -m vulture $(ARGS) # =================================================================== -# GIT +# Fixers # =================================================================== -# git-tag a new release -git-tag-release: - git tag -a release-`python -c "import setup; print(setup.get_version())"` -m `git rev-list HEAD --count`:`git rev-parse --short HEAD` - git push --follow-tags +fix-black: ## Reformat python code with black. + @$(call _ls,'*.py') | xargs $(PYTHON) -m black -# install GIT pre-commit hook -install-git-hooks: - ln -sf ../../.git-pre-commit .git/hooks/pre-commit - chmod +x .git/hooks/pre-commit +fix-ruff: ## Fix ruff errors. + @$(call _ls,'*.py') | xargs $(PYTHON) -m ruff check --fix --output-format=concise $(ARGS) + +fix-c: ## Reformat C code with clang-format. + @$(call _ls,'*.c' '*.h') | xargs -P0 -I{} clang-format -i {} # parallel exec + +fix-toml: ## Fix pyproject.toml + @$(call _ls,'*.toml') | xargs toml-sort + +fix-rst: ## Re-wrap .rst files. + @$(call _ls,'*.rst') | xargs rstwrap + +fix-dprint: ## Reformat .md / .json / .yml / .js / .css files. + @$(DPRINT) fmt + +fix-all: ## Run all code fixers. + $(MAKE) fix-ruff + $(MAKE) fix-black + $(MAKE) fix-c + $(MAKE) fix-rst + $(MAKE) fix-toml + $(MAKE) fix-dprint + +# =================================================================== +# CI jobs +# =================================================================== + +ci-lint: ## Run all linters on GitHub CI. + $(MAKE) install-pydeps-lint + test -x $(DPRINT) || curl -fsSL https://dprint.dev/install.sh | sh + $(DPRINT) --version + clang-format --version + $(MAKE) lint-all + +ci-test: ## Run tests on GitHub CI. + $(MAKE) install-sysdeps + # Editable install: it builds in-place, and having psutil already + # installed stops pip from pulling it from PyPI for psleak. + $(INSTALL_PYDEPS) --editable . + $(MAKE) install-pydeps-test + $(MAKE) print-sysinfo + # Warm pywin32's gen_py cache: concurrent first imports of wmi in + # the pytest workers corrupt it (EOFError from gencache). + if [ "$$OS" = "Windows_NT" ]; then $(PYTHON) -c "import wmi"; fi + $(MAKE) test-parallel + +ci-check-dist: ## Run all sanity checks re. to the package distribution. + $(INSTALL_PYDEPS) setuptools virtualenv twine check-manifest validate-pyproject[all] abi3audit + $(MAKE) create-sdist + mv wheelhouse/* dist/ + $(MAKE) check-dist + $(PYTHON) scripts/internal/print_dist.py --check # =================================================================== # Distribution # =================================================================== -# Upload source tarball on https://pypi.python.org/pypi/psutil. -upload-src: clean - $(PYTHON) setup.py sdist upload +# --- create + +generate-manifest: ## Generates MANIFEST.in file. + $(PYTHON) scripts/internal/generate_manifest.py > MANIFEST.in + +create-sdist: ## Create tar.gz source distribution. + $(MAKE) generate-manifest + $(PYTHON_ENV_VARS) $(PYTHON) setup.py sdist + +create-wheels: ## Create .whl files + $(PYTHON_ENV_VARS) $(PYTHON) setup.py bdist_wheel + +download-wheels: ## Download latest wheels hosted on github. + $(PYTHON) scripts/internal/download_wheels.py --tokenfile=~/.github.api.key + $(MAKE) print-dist + +create-dist: ## Create .tar.gz + .whl distribution. + $(MAKE) create-sdist + $(MAKE) download-wheels + +# --- check + +check-manifest: ## Check sanity of MANIFEST.in file. + $(PYTHON) -m check_manifest -v + +check-pyproject: ## Check sanity of pyproject.toml file. + $(PYTHON) -m validate_pyproject -v pyproject.toml + +check-sdist: ## Check sanity of source distribution. + $(PYTHON_ENV_VARS) $(PYTHON) -m virtualenv --clear --no-wheel --quiet build/venv + $(PYTHON_ENV_VARS) build/venv/bin/python -m pip install -v --isolated --quiet dist/*.tar.gz + $(PYTHON_ENV_VARS) build/venv/bin/python -c "import os; os.chdir('build/venv'); import psutil" + $(PYTHON) -m twine check --strict dist/*.tar.gz + +check-wheels: ## Check sanity of wheels. + $(PYTHON) -m abi3audit --verbose --strict dist/*-abi3-*.whl + $(PYTHON) -m twine check --strict dist/*.whl + +check-dist: ## Run all sanity checks re. to the package distribution. + $(MAKE) -j \ + check-manifest \ + check-pyproject \ + check-sdist \ + check-wheels + +# --- release + +pre-release: ## Check if we're ready to produce a new release. + $(MAKE) clean + $(MAKE) create-dist + $(MAKE) check-dist + $(MAKE) install + @$(PYTHON) -c \ + "import requests, sys; \ + from packaging.version import parse; \ + from psutil import __version__; \ + res = requests.get('https://pypi.org/pypi/psutil/json', timeout=5); \ + versions = sorted(res.json()['releases'], key=parse, reverse=True); \ + sys.exit('version %r already exists on PYPI' % __version__) if __version__ in versions else 0" + @ver=$$($(PYTHON) -c "from psutil import __version__; print(__version__)"); \ + grep -q "$$ver" docs/changelog.rst || { echo "ERR: version $$ver not found in docs/changelog.rst"; exit 1; }; \ + grep -q "$$ver" docs/timeline.rst || { echo "ERR: version $$ver not found in docs/timeline.rst"; exit 1; } + $(MAKE) print-hashes + $(MAKE) print-dist + +release: ## Upload a new release. + $(PYTHON) -m twine upload dist/*.tar.gz + $(PYTHON) -m twine upload dist/*.whl + $(MAKE) git-tag-release + +git-tag-release: ## Git-tag a new release. + git tag -a v`$(PYTHON) -c "import setup; print(setup.get_version())"` -m `git rev-list HEAD --count`:`git rev-parse --short HEAD` + git push --follow-tags + +# =================================================================== +# Printers +# =================================================================== + +print-announce: ## Print announce of new release. + @$(PYTHON) scripts/internal/print_announce.py + +print-access-denied: ## Print AD exceptions + $(PYTHON) scripts/internal/print_access_denied.py + +print-api-speed: ## Benchmark all API calls + $(PYTHON) scripts/internal/print_api_speed.py $(ARGS) + +print-downloads: ## Print PYPI download statistics + $(PYTHON) scripts/internal/print_downloads.py + +print-hashes: ## Prints hashes of files in dist/ directory + $(PYTHON) scripts/internal/print_hashes.py + +print-sysinfo: ## Prints system info + $(PYTHON) scripts/internal/print_sysinfo.py + +print-dist: ## Print downloaded wheels / tar.gz + $(PYTHON) scripts/internal/print_dist.py + +# =================================================================== +# Misc +# =================================================================== + +grep-todos: ## Look for TODOs in the source files. + git grep -EIn "TODO|FIXME|XXX" + +bench-oneshot: ## Benchmarks for oneshot() ctx manager (see #799). + $(PYTHON) scripts/internal/bench_oneshot.py + +bench-oneshot-2: ## Same as above but using perf module (more precise). + $(PYTHON) scripts/internal/bench_oneshot_2.py + +find-broken-links: ## Look for broken links in source files (except docs/). + git ls-files | grep -v '^docs/' | xargs $(PYTHON) -Wa scripts/internal/find_broken_links.py -# Build and upload doc on https://pythonhosted.org/psutil/. -# Requires "pip install sphinx-pypi-upload". -upload-doc: - cd docs; make html - $(PYTHON) setup.py upload_sphinx --upload-dir=docs/_build/html +_CI_JOBS = $(patsubst .github/workflows/%.yml,%,$(shell grep -l workflow_dispatch .github/workflows/*.yml)) -# download exes/wheels hosted on appveyor -win-download-exes: - $(PYTHON) .ci/appveyor/download_exes.py --user giampaolo --project psutil +ci-run: ## Manually run a CI workflow, e.g. `make ci-run JOB=bsd` + @echo "$(_CI_JOBS)" | tr ' ' '\n' | grep -qx "$(JOB)" || { echo "Usage: make ci-run JOB=<$$(echo $(_CI_JOBS) | tr ' ' '|')>"; exit 1; } + gh workflow run $(JOB).yml --ref $$(git rev-parse --abbrev-ref HEAD) -# upload exes/wheels in dist/* directory to PYPI -win-upload-exes: - $(PYTHON) -m twine upload dist/* +help: ## Display callable targets. + @awk -F':.*?## ' '/^[a-zA-Z0-9_.-]+:.*?## / {printf "\033[36m%-24s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort diff --git a/README.rst b/README.rst index eed2483272..ed6f6777cd 100644 --- a/README.rst +++ b/README.rst @@ -1,407 +1,192 @@ -.. image:: https://img.shields.io/travis/giampaolo/psutil/master.svg?maxAge=3600&label=Linux%20/%20OSX - :target: https://travis-ci.org/giampaolo/psutil - :alt: Linux tests (Travis) - -.. image:: https://img.shields.io/appveyor/ci/giampaolo/psutil/master.svg?maxAge=3600&label=Windows - :target: https://ci.appveyor.com/project/giampaolo/psutil - :alt: Windows tests (Appveyor) - -.. image:: https://coveralls.io/repos/github/giampaolo/psutil/badge.svg?branch=master - :target: https://coveralls.io/github/giampaolo/psutil?branch=master - :alt: Test coverage (coverall.io) - -.. image:: https://img.shields.io/pypi/v/psutil.svg?label=version - :target: https://pypi.python.org/pypi/psutil/ - :alt: Latest version - -.. image:: https://img.shields.io/github/stars/giampaolo/psutil.svg - :target: https://github.com/giampaolo/psutil/ - :alt: Github stars - -.. image:: https://img.shields.io/pypi/l/psutil.svg - :target: https://pypi.python.org/pypi/psutil/ - :alt: License - -=========== -Quick links -=========== - -- `Home page `_ -- `Install `_ -- `Documentation `_ -- `Download `_ -- `Forum `_ -- `Blog `_ -- `Development guide `_ -- `What's new `_ +.. -======= -Summary +.. raw:: html + +
+ psutil +

Process and System Utilities for Python

+ Documentation    + Blog    + Who uses psutil    +
+ +
+ +
+ Downloads + + Binary packages + + Latest version +
+ +
+ Linux, macOS, Windows + + FreeBSD, NetBSD, OpenBSD + + Documentation +
+ +..
+ +About +===== + +psutil is a cross-platform library for retrieving information about running +**processes** and **system utilization** (CPU, memory, disks, network, sensors) +in Python. It is useful mainly for **system monitoring**, **profiling**, +**limiting process resources**, and **managing running processes**. It +implements many functionalities offered by UNIX command line tool such as +*ps, top, free, iotop, netstat, ifconfig, lsof* and others (see +`shell equivalents`_). Psutil supports the following platforms: + +- **Linux** +- **Windows** +- **macOS** +- **FreeBSD, OpenBSD**, **NetBSD** +- **Sun Solaris** +- **AIX** + +Adoption +======== + +psutil is among the +`top 100 `__ most-downloaded +packages on PyPI, with **390+ million** downloads per month and **780,000+** +`GitHub repositories `__ +using it. See also `adoptions `__ and +`alternatives `__. + +Install ======= -psutil (process and system utilities) is a cross-platform library for -retrieving information on **running processes** and **system utilization** -(CPU, memory, disks, network) in Python. It is useful mainly for **system -monitoring**, **profiling and limiting process resources** and **management of -running processes**. It implements many functionalities offered by command line -tools such as: ps, top, lsof, netstat, ifconfig, who, df, kill, free, nice, -ionice, iostat, iotop, uptime, pidof, tty, taskset, pmap. It currently supports -**Linux, Windows, OSX, Sun Solaris, FreeBSD, OpenBSD** and **NetBSD**, -both **32-bit** and **64-bit** architectures, with Python versions from **2.6 -to 3.5** (users of Python 2.4 and 2.5 may use -`2.1.3 `__ version). -`PyPy `__ is also known to work. - -==================== -Example applications -==================== - -- https://github.com/nicolargo/glances -- https://github.com/google/grr -- https://github.com/Jahaja/psdash -- https://github.com/giampaolo/psutil/tree/master/scripts +.. code-block:: + + pip install psutil + +For platform-specific details see `installation `_. + +Documentation +============= + +psutil documentation is available at https://psutil.io. + +.. + +Sponsors +======== + +.. raw:: html + + + + + + + + + + + +.. -============== Example usages ============== -CPU -=== +For the full API with more examples, see the +`API overview `_ and +`API reference `_. + +**CPU** .. code-block:: python >>> import psutil - >>> psutil.cpu_times() - scputimes(user=3961.46, nice=169.729, system=2150.659, idle=16900.540, iowait=629.59, irq=0.0, softirq=19.42, steal=0.0, guest=0, nice=0.0) - >>> - >>> for x in range(3): - ... psutil.cpu_percent(interval=1) - ... - 4.0 - 5.9 - 3.8 - >>> - >>> for x in range(3): - ... psutil.cpu_percent(interval=1, percpu=True) - ... + >>> psutil.cpu_percent(interval=1, percpu=True) [4.0, 6.9, 3.7, 9.2] - [7.0, 8.5, 2.4, 2.1] - [1.2, 9.0, 9.9, 7.2] - >>> - >>> for x in range(3): - ... psutil.cpu_times_percent(interval=1, percpu=False) - ... - scputimes(user=1.5, nice=0.0, system=0.5, idle=96.5, iowait=1.5, irq=0.0, softirq=0.0, steal=0.0, guest=0.0, guest_nice=0.0) - scputimes(user=1.0, nice=0.0, system=0.0, idle=99.0, iowait=0.0, irq=0.0, softirq=0.0, steal=0.0, guest=0.0, guest_nice=0.0) - scputimes(user=2.0, nice=0.0, system=0.0, idle=98.0, iowait=0.0, irq=0.0, softirq=0.0, steal=0.0, guest=0.0, guest_nice=0.0) - >>> - >>> psutil.cpu_count() - 4 >>> psutil.cpu_count(logical=False) 2 - >>> - >>> psutil.cpu_stats() - scpustats(ctx_switches=20455687, interrupts=6598984, soft_interrupts=2134212, syscalls=0) + >>> psutil.cpu_freq() + scpufreq(current=931.42, min=800.0, max=3500.0) -Memory -====== +**Memory** .. code-block:: python >>> psutil.virtual_memory() - svmem(total=10367352832, available=6472179712, percent=37.6, used=8186245120, free=2181107712, active=4748992512, inactive=2758115328, buffers=790724608, cached=3500347392, shared=787554304) + svmem(total=10367352832, available=6472179712, percent=37.6, used=8186245120, free=2181107712, ...) >>> psutil.swap_memory() sswap(total=2097147904, used=296128512, free=1801019392, percent=14.1, sin=304193536, sout=677842944) - >>> -Disks -===== +**Disks** .. code-block:: python >>> psutil.disk_partitions() [sdiskpart(device='/dev/sda1', mountpoint='/', fstype='ext4', opts='rw,nosuid'), - sdiskpart(device='/dev/sda2', mountpoint='/home', fstype='ext, opts='rw')] - >>> + sdiskpart(device='/dev/sda2', mountpoint='/home', fstype='ext', opts='rw')] >>> psutil.disk_usage('/') sdiskusage(total=21378641920, used=4809781248, free=15482871808, percent=22.5) - >>> - >>> psutil.disk_io_counters(perdisk=False) - sdiskio(read_count=719566, write_count=1082197, read_bytes=18626220032, write_bytes=24081764352, read_time=5023392, write_time=63199568, read_merged_count=619166, write_merged_count=812396, busy_time=4523412) - >>> -Network -======= +**Network** .. code-block:: python >>> psutil.net_io_counters(pernic=True) - {'eth0': netio(bytes_sent=485291293, bytes_recv=6004858642, packets_sent=3251564, packets_recv=4787798, errin=0, errout=0, dropin=0, dropout=0), - 'lo': netio(bytes_sent=2838627, bytes_recv=2838627, packets_sent=30567, packets_recv=30567, errin=0, errout=0, dropin=0, dropout=0)} - >>> - >>> psutil.net_connections() - [pconn(fd=115, family=, type=, laddr=('10.0.0.1', 48776), raddr=('93.186.135.91', 80), status='ESTABLISHED', pid=1254), - pconn(fd=117, family=, type=, laddr=('10.0.0.1', 43761), raddr=('72.14.234.100', 80), status='CLOSING', pid=2987), - pconn(fd=-1, family=, type=, laddr=('10.0.0.1', 60759), raddr=('72.14.234.104', 80), status='ESTABLISHED', pid=None), - pconn(fd=-1, family=, type=, laddr=('10.0.0.1', 51314), raddr=('72.14.234.83', 443), status='SYN_SENT', pid=None) + {'eth0': netio(bytes_sent=485291293, bytes_recv=6004858642, ...), + 'lo': netio(bytes_sent=2838627, bytes_recv=2838627, ...)} + >>> psutil.net_connections(kind='tcp') + [sconn(fd=115, family=2, type=1, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED', pid=1254), ...] - >>> - >>> psutil.net_if_addrs() - {'lo': [snic(family=, address='127.0.0.1', netmask='255.0.0.0', broadcast='127.0.0.1', ptp=None), - snic(family=, address='::1', netmask='ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', broadcast=None, ptp=None), - snic(family=, address='00:00:00:00:00:00', netmask=None, broadcast='00:00:00:00:00:00', ptp=None)], - 'wlan0': [snic(family=, address='192.168.1.3', netmask='255.255.255.0', broadcast='192.168.1.255', ptp=None), - snic(family=, address='fe80::c685:8ff:fe45:641%wlan0', netmask='ffff:ffff:ffff:ffff::', broadcast=None, ptp=None), - snic(family=, address='c4:85:08:45:06:41', netmask=None, broadcast='ff:ff:ff:ff:ff:ff', ptp=None)]} - >>> - >>> psutil.net_if_stats() - {'eth0': snicstats(isup=True, duplex=, speed=100, mtu=1500), - 'lo': snicstats(isup=True, duplex=, speed=0, mtu=65536)} -Other system info -================= +**Sensors** .. code-block:: python - >>> psutil.users() - [user(name='giampaolo', terminal='pts/2', host='localhost', started=1340737536.0), - user(name='giampaolo', terminal='pts/3', host='localhost', started=1340737792.0)] - >>> - >>> psutil.boot_time() - 1365519115.0 - >>> + >>> psutil.sensors_temperatures() + {'coretemp': [shwtemp(label='Physical id 0', current=52.0, high=100.0, critical=100.0), + shwtemp(label='Core 0', current=45.0, high=100.0, critical=100.0)]} + >>> psutil.sensors_battery() + sbattery(percent=93, secsleft=16628, power_plugged=False) -Process management -================== +**Processes** .. code-block:: python - >>> import psutil - >>> psutil.pids() - [1, 2, 3, 4, 5, 6, 7, 46, 48, 50, 51, 178, 182, 222, 223, 224, - 268, 1215, 1216, 1220, 1221, 1243, 1244, 1301, 1601, 2237, 2355, - 2637, 2774, 3932, 4176, 4177, 4185, 4187, 4189, 4225, 4243, 4245, - 4263, 4282, 4306, 4311, 4312, 4313, 4314, 4337, 4339, 4357, 4358, - 4363, 4383, 4395, 4408, 4433, 4443, 4445, 4446, 5167, 5234, 5235, - 5252, 5318, 5424, 5644, 6987, 7054, 7055, 7071] - >>> >>> p = psutil.Process(7055) >>> p.name() - 'python' + 'python3' >>> p.exe() - '/usr/bin/python' - >>> p.cwd() - '/home/giampaolo' - >>> p.cmdline() - ['/usr/bin/python', 'main.py'] - >>> - >>> p.status() - 'running' - >>> p.username() - 'giampaolo' - >>> p.create_time() - 1267551141.5019531 - >>> p.terminal() - '/dev/pts/0' - >>> - >>> p.uids() - puids(real=1000, effective=1000, saved=1000) - >>> p.gids() - pgids(real=1000, effective=1000, saved=1000) - >>> - >>> p.cpu_times() - pcputimes(user=1.02, system=0.31, children_user=0.32, children_system=0.1) + '/usr/bin/python3' >>> p.cpu_percent(interval=1.0) 12.1 - >>> p.cpu_affinity() - [0, 1, 2, 3] - >>> p.cpu_affinity([0]) # set - >>> - >>> p.memory_percent() - 0.63423 - >>> >>> p.memory_info() - pmem(rss=10915840, vms=67608576, shared=3313664, text=2310144, lib=0, data=7262208, dirty=0) - >>> - >>> p.memory_full_info() # "real" USS memory usage (Linux, OSX, Win only) - pfullmem(rss=10199040, vms=52133888, shared=3887104, text=2867200, lib=0, data=5967872, dirty=0, uss=6545408, pss=6872064, swap=0) - >>> - >>> p.memory_maps() - [pmmap_grouped(path='/lib/x8664-linux-gnu/libutil-2.15.so', rss=32768, size=2125824, pss=32768, shared_clean=0, shared_dirty=0, private_clean=20480, private_dirty=12288, referenced=32768, anonymous=12288, swap=0), - pmmap_grouped(path='/lib/x8664-linux-gnu/libc-2.15.so', rss=3821568, size=3842048, pss=3821568, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=3821568, referenced=3575808, anonymous=3821568, swap=0), - pmmap_grouped(path='/lib/x8664-linux-gnu/libcrypto.so.0.1', rss=34124, rss=32768, size=2134016, pss=15360, shared_clean=24576, shared_dirty=0, private_clean=0, private_dirty=8192, referenced=24576, anonymous=8192, swap=0), - pmmap_grouped(path='[heap]', rss=32768, size=139264, pss=32768, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=32768, referenced=32768, anonymous=32768, swap=0), - pmmap_grouped(path='[stack]', rss=2465792, size=2494464, pss=2465792, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=2465792, referenced=2277376, anonymous=2465792, swap=0), - ...] - >>> - >>> p.io_counters() - pio(read_count=478001, write_count=59371, read_bytes=700416, write_bytes=69632) - >>> + pmem(rss=3164160, vms=4410163, shared=897433, text=302694, data=2422374) + >>> p.net_connections(kind='tcp') + [pconn(fd=115, family=2, type=1, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED')] >>> p.open_files() - [popenfile(path='/home/giampaolo/svn/psutil/setup.py', fd=3, position=0, mode='r', flags=32768), - popenfile(path='/var/log/monitd', fd=4, position=235542, mode='a', flags=33793)] - >>> - >>> p.connections() - [pconn(fd=115, family=, type=, laddr=('10.0.0.1', 48776), raddr=('93.186.135.91', 80), status='ESTABLISHED'), - pconn(fd=117, family=, type=, laddr=('10.0.0.1', 43761), raddr=('72.14.234.100', 80), status='CLOSING'), - pconn(fd=119, family=, type=, laddr=('10.0.0.1', 60759), raddr=('72.14.234.104', 80), status='ESTABLISHED'), - pconn(fd=123, family=, type=, laddr=('10.0.0.1', 51314), raddr=('72.14.234.83', 443), status='SYN_SENT')] - >>> - >>> p.num_threads() - 4 - >>> p.num_fds() - 8 - >>> p.threads() - [pthread(id=5234, user_time=22.5, system_time=9.2891), - pthread(id=5235, user_time=0.0, system_time=0.0), - pthread(id=5236, user_time=0.0, system_time=0.0), - pthread(id=5237, user_time=0.0707, system_time=1.1)] - >>> - >>> p.num_ctx_switches() - pctxsw(voluntary=78, involuntary=19) - >>> - >>> p.nice() - 0 - >>> p.nice(10) # set - >>> - >>> p.ionice(psutil.IOPRIO_CLASS_IDLE) # IO priority (Win and Linux only) - >>> p.ionice() - pionice(ioclass=, value=0) - >>> - >>> p.rlimit(psutil.RLIMIT_NOFILE, (5, 5)) # set resource limits (Linux only) - >>> p.rlimit(psutil.RLIMIT_NOFILE) - (5, 5) - >>> - >>> p.environ() - {'LC_PAPER': 'it_IT.UTF-8', 'SHELL': '/bin/bash', 'GREP_OPTIONS': '--color=auto', - 'XDG_CONFIG_DIRS': '/etc/xdg/xdg-ubuntu:/usr/share/upstart/xdg:/etc/xdg', 'COLORTERM': 'gnome-terminal', - ...} - >>> - >>> p.suspend() - >>> p.resume() + [popenfile(path='/home/giampaolo/monit.py', fd=3, position=0, mode='r', flags=32768)] >>> - >>> p.terminate() - >>> p.wait(timeout=3) - 0 - >>> - >>> psutil.test() - USER PID %CPU %MEM VSZ RSS TTY START TIME COMMAND - root 1 0.0 0.0 24584 2240 Jun17 00:00 init - root 2 0.0 0.0 0 0 Jun17 00:00 kthreadd - root 3 0.0 0.0 0 0 Jun17 00:05 ksoftirqd/0 + >>> for p in psutil.process_iter(['pid', 'name']): + ... print(p.pid, p.name()) ... - giampaolo 31475 0.0 0.0 20760 3024 /dev/pts/0 Jun19 00:00 python2.4 - giampaolo 31721 0.0 2.2 773060 181896 00:04 10:30 chrome - root 31763 0.0 0.0 0 0 00:05 00:00 kworker/0:1 - >>> - -Further process APIs -==================== - -.. code-block:: python - - >>> for p in psutil.process_iter(): - ... print(p) + 1 systemd + 2 kthreadd + 3 ksoftirqd/0 ... - psutil.Process(pid=1, name='init') - psutil.Process(pid=2, name='kthreadd') - psutil.Process(pid=3, name='ksoftirqd/0') - ... - >>> - >>> def on_terminate(proc): - ... print("process {} terminated".format(proc)) - ... - >>> # waits for multiple processes to terminate - >>> gone, alive = psutil.wait_procs(procs_list, timeout=3, callback=on_terminate) - >>> - -Windows services -================ -.. code-block:: python - - >>> list(psutil.win_service_iter()) - [, - , - , - , - ...] - >>> s = psutil.win_service_get('alg') - >>> s.as_dict() - {'binpath': 'C:\\Windows\\System32\\alg.exe', - 'description': 'Provides support for 3rd party protocol plug-ins for Internet Connection Sharing', - 'display_name': 'Application Layer Gateway Service', - 'name': 'alg', - 'pid': None, - 'start_type': 'manual', - 'status': 'stopped', - 'username': 'NT AUTHORITY\\LocalService'} - -====== -Donate -====== - -A lot of time and effort went into making psutil as it is right now. -If you feel psutil is useful to you or your business and want to support its future development please consider donating me (`Giampaolo Rodola' `_) some money. -I only ask for a small donation, but of course I appreciate any amount. - -.. image:: http://www.paypal.com/en_US/i/btn/x-click-but04.gif - :target: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A9ZS7PKKRM3S8 - :alt: Donate via PayPal +.. _`shell equivalents`: https://psutil.io/shell-equivalents/ -Don't want to donate money? Then maybe you could `write me a recommendation on Linkedin `_. +.. -============ -Mailing list -============ - -http://groups.google.com/group/psutil/ - -======== -Timeline -======== +License +======= -- 2016-06-18: `psutil-4.3.0.tar.gz `_ -- 2016-05-15: `psutil-4.2.0.tar.gz `_ -- 2016-03-12: `psutil-4.1.0.tar.gz `_ -- 2016-02-17: `psutil-4.0.0.tar.gz `_ -- 2016-01-20: `psutil-3.4.2.tar.gz `_ -- 2016-01-15: `psutil-3.4.1.tar.gz `_ -- 2015-11-25: `psutil-3.3.0.tar.gz `_ -- 2015-10-04: `psutil-3.2.2.tar.gz `_ -- 2015-09-03: `psutil-3.2.1.tar.gz `_ -- 2015-09-02: `psutil-3.2.0.tar.gz `_ -- 2015-07-15: `psutil-3.1.1.tar.gz `_ -- 2015-07-15: `psutil-3.1.0.tar.gz `_ -- 2015-06-18: `psutil-3.0.1.tar.gz `_ -- 2015-06-13: `psutil-3.0.0.tar.gz `_ -- 2015-02-02: `psutil-2.2.1.tar.gz `_ -- 2015-01-06: `psutil-2.2.0.tar.gz `_ -- 2014-09-26: `psutil-2.1.3.tar.gz `_ -- 2014-09-21: `psutil-2.1.2.tar.gz `_ -- 2014-04-30: `psutil-2.1.1.tar.gz `_ -- 2014-04-08: `psutil-2.1.0.tar.gz `_ -- 2014-03-10: `psutil-2.0.0.tar.gz `_ -- 2013-11-25: `psutil-1.2.1.tar.gz `_ -- 2013-11-20: `psutil-1.2.0.tar.gz `_ -- 2013-11-07: `psutil-1.1.3.tar.gz `_ -- 2013-10-22: `psutil-1.1.2.tar.gz `_ -- 2013-10-08: `psutil-1.1.1.tar.gz `_ -- 2013-09-28: `psutil-1.1.0.tar.gz `_ -- 2013-07-12: `psutil-1.0.1.tar.gz `_ -- 2013-07-10: `psutil-1.0.0.tar.gz `_ -- 2013-05-03: `psutil-0.7.1.tar.gz `_ -- 2013-04-12: `psutil-0.7.0.tar.gz `_ -- 2012-08-16: `psutil-0.6.1.tar.gz `_ -- 2012-08-13: `psutil-0.6.0.tar.gz `_ -- 2012-06-29: `psutil-0.5.1.tar.gz `_ -- 2012-06-27: `psutil-0.5.0.tar.gz `_ -- 2011-12-14: `psutil-0.4.1.tar.gz `_ -- 2011-10-29: `psutil-0.4.0.tar.gz `_ -- 2011-07-08: `psutil-0.3.0.tar.gz `_ -- 2011-03-20: `psutil-0.2.1.tar.gz `_ -- 2010-11-13: `psutil-0.2.0.tar.gz `_ -- 2010-03-02: `psutil-0.1.3.tar.gz `_ -- 2009-05-06: `psutil-0.1.2.tar.gz `_ -- 2009-03-06: `psutil-0.1.1.tar.gz `_ -- 2009-01-27: `psutil-0.1.0.tar.gz `_ +BSD-3 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..fda8e36955 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,10 @@ +# Security Policy + +If you have discovered a security vulnerability in this project, please report +it privately. **Do not disclose it as a public issue**. This gives me time to +fix the issue before public exposure, reducing the chance that an exploit will +be used before a patch is released. + +To report a security vulnerability use the +[Tidelift security contact](https://tidelift.com/security). Tidelift will +coordinate the fix and the disclosure of the reported problem. diff --git a/_bootstrap.py b/_bootstrap.py new file mode 100644 index 0000000000..7a1da56fd4 --- /dev/null +++ b/_bootstrap.py @@ -0,0 +1,41 @@ +# Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Bootstrap utilities for loading psutil modules without psutil +being installed. +""" + +import ast +import importlib.util +import os +import pathlib + +ROOT_DIR = pathlib.Path(__file__).resolve().parent + + +def load_module(path): + """Load a Python module by file path without importing it + as part of a package. + """ + name = os.path.splitext(os.path.basename(path))[0] + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def get_version(): + """Extract __version__ from psutil/__init__.py using AST + (no imports needed). + """ + path = ROOT_DIR / "psutil" / "__init__.py" + with open(path, encoding="utf-8") as f: + mod = ast.parse(f.read()) + for node in mod.body: + if isinstance(node, ast.Assign): + for target in node.targets: + if getattr(target, "id", None) == "__version__": + return ast.literal_eval(node.value) + msg = "could not find __version__" + raise RuntimeError(msg) diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 5299720454..0000000000 --- a/appveyor.yml +++ /dev/null @@ -1,93 +0,0 @@ -os: Visual Studio 2015 - -environment: - - global: - # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the - # /E:ON and /V:ON options are not enabled in the batch script intepreter - # See: http://stackoverflow.com/a/13751649/163740 - WITH_COMPILER: "cmd /E:ON /V:ON /C .\\.ci\\appveyor\\run_with_compiler.cmd" - - matrix: - # Pre-installed Python versions, which Appveyor may upgrade to - # a later point release. - - # 32 bits - - - PYTHON: "C:\\Python27" - PYTHON_VERSION: "2.7.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python33" - PYTHON_VERSION: "3.3.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python34" - PYTHON_VERSION: "3.4.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python35" - PYTHON_VERSION: "3.5.x" - PYTHON_ARCH: "32" - - # 64 bits - - - PYTHON: "C:\\Python27-x64" - PYTHON_VERSION: "2.7.x" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python33-x64" - PYTHON_VERSION: "3.3.x" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4.x" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5.x" - PYTHON_ARCH: "64" - ARCH: x86_64 - VS_VER: "2015" - INSTANCENAME: "SQL2012SP1" - - # Also build on a Python version not pre-installed by Appveyor. - # See: https://github.com/ogrisel/python-appveyor-demo/issues/10 - - # - PYTHON: "C:\\Python266" - # PYTHON_VERSION: "2.6.6" - # PYTHON_ARCH: "32" - -init: - - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" - -install: - - "powershell .ci\\appveyor\\install.ps1" - # - ps: (new-object net.webclient).DownloadFile('https://raw.github.com/pypa/pip/master/contrib/get-pip.py', 'C:/get-pip.py') - - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32 wmi wheel" - - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" - - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build" - - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build build_ext -i" - - "%WITH_COMPILER% %PYTHON%/python.exe setup.py develop" - # 1.0.1 is the latest release supporting python 2.6 - - "%WITH_COMPILER% %PYTHON%/Scripts/pip.exe install mock==1.0.1" - -build: off - -test_script: - - "%WITH_COMPILER% %PYTHON%/python -V" - - "%WITH_COMPILER% %PYTHON%/python psutil/tests/runner.py" - -after_test: - - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" - - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wininst" - -artifacts: - - path: dist\* - -# on_success: -# - might want to upload the content of dist/*.whl to a public wheelhouse - -skip_commits: - message: skip-ci diff --git a/docs/404.rst b/docs/404.rst new file mode 100644 index 0000000000..a5e981cc3d --- /dev/null +++ b/docs/404.rst @@ -0,0 +1,26 @@ +:orphan: + +404 - Page not found +==================== + +.. code-block:: pytb + + Traceback (most recent call last): + File "", line 1, in + fetch_page(url) + psutil.NoSuchProcess: process no longer exists (pid=404) + +The page you're looking for doesn't exist. Or it does, as a +:class:`~psutil.ZombieProcess` we can't reap. + +What now? +--------- + +- Hit ``Ctrl + K`` (or ``Cmd + K``) to open the search box. +- Jump to the :doc:`install guide `, :doc:`API reference `, + :doc:`FAQ ` or :doc:`recipes `. +- Read the :doc:`blog ` for posts and release notes. +- If you got here from a link that *should* work, please + `open an issue on GitHub`_. + +.. _open an issue on GitHub: https://github.com/giampaolo/psutil/issues/new?title=Broken+link%3A+ diff --git a/docs/DEVNOTES.md b/docs/DEVNOTES.md new file mode 100644 index 0000000000..7382ca785a --- /dev/null +++ b/docs/DEVNOTES.md @@ -0,0 +1,223 @@ +A collection of ideas and notes about stuff to implement in future versions. + +## Inconsistencies + +(the "too late" section) + +- `PROCFS_PATH` should have been `set_procfs_path()`. + +- `virtual_memory()` should have been `memory_virtual()`. + +- `swap_memory()` should have been `memory_swap()`. + +- Named tuples are problematic. Positional unpacking of named tuples could be + deprecated. Return frozen dataclasses (with a common base class) instead of + `typing.NamedTuple`. The base class would keep `_fields`, `_asdict()` and + `len()` for compat, but `__iter__` and `__getitem__` would emit + DeprecationWarning. Main concern: `isinstance(x, tuple)` would break. + +## Rejected ideas + +- #550: threads per core +- #1667: `process_iter(new_only=True)` + +## Features + +- #2794: enrich `Process.wait()` return value with exit code enums. + +- `net_if_addrs()` could return AF_BLUETOOTH interfaces. E.g. + https://pypi.org/project/netifaces does this. + +- Use `resource.getrusage()` to get current process CPU times (it has more + precision). + +- (UNIX) `Process.root()` (different from `cwd()`). + +- (Linux) locked files via /proc/locks: + https://www.centos.org/docs/5/html/5.2/Deployment_Guide/s2-proc-locks.html + +- #269: NIC rx/tx queue. This should probably go into `net_if_stats()`. Figure + out on what platforms this is supported. Linux: yes. Others? + +- Asynchronous `psutil.Popen` (see http://bugs.python.org/issue1191964) + +- (Windows) fall back on using WMIC for Process methods returning + `AccessDenied`. + +- #613: thread names; patch for macOS available at: + https://code.google.com/p/plcrashreporter/issues/detail?id=65 Sample code: + https://github.com/janmojzis/pstree/blob/master/proc_kvm.c + +- `scripts/taskmgr-gui.py` (using tk). + +- system-wide number of open file descriptors: + - https://jira.hyperic.com/browse/SIGAR-30 + +- Number of system threads. + - Windows: + http://msdn.microsoft.com/en-us/library/windows/desktop/ms684824(v=vs.85).aspx + +- `psutil.proc_tree()` something which obtains a `{pid:ppid, ...}` struct for + all running processes in one shot. This can be factored out from + `Process.children()` and exposed as a first class function. PROS: on Windows + we can take advantage of `ppid_map()`, which is faster than iterating over + all PIDs. CONS: `scripts/pstree.py` shows this can be easily done in the user + code, so maybe it's not worth the addition. + +- advanced cmdline interface exposing the whole API and providing different + kind of outputs (e.g. pprinted, colorized, json). + +- Linux: process cgroups (http://en.wikipedia.org/wiki/Cgroups). They look + similar to `prlimit()` in terms of functionality but, uglier (they should + allow limiting per-process network IO resources though, which is great). + Needs further reading. + +- Python 3.3. exposed different sched.h functions: + http://docs.python.org/dev/whatsnew/3.3.html#os + http://bugs.python.org/issue12655 + http://docs.python.org/dev/library/os.html#interface-to-the-scheduler It It + might be worth to take a look and figure out whether we can include some of + those in psutil. + +- `os.times()` provides `elapsed` times (`Process.cpu_times()` might as well?). + +- Enrich exception classes hierarchy on Python >= 3.3 / post PEP-3151 so that: + - `NoSuchProcess` inherits from `ProcessLookupError` + - `AccessDenied` inherits from `PermissionError` + - `TimeoutExpired inherits` from TimeoutError (debatable) See: + http://docs.python.org/3/library/exceptions.html#os-exceptions + +- `Process.threads()` might grow an extra "id" parameter so that it can be used + as: + + ```python + >>> p = psutil.Process(os.getpid()) + >>> p.threads(id=psutil.current_thread_id()) + thread(id=2539, user_time=0.03, system_time=0.02) + >>> + ``` + + Note: this leads to questions such as "should we have a custom `NoSuchThread` + exception? Also see issue #418. Also note: this would work with `os.getpid()` + only. `psutil.current_thread_id()` might be desirable as per issue #418 + though. + +- should `TimeoutExpired` exception have a 'msg' kwarg similar to + `NoSuchProcess` and `AccessDenied`? Not that we need it, but currently we + cannot raise a `TimeoutExpired` exception with a specific error string. + +- round `Process.memory_percent() result? + +## Resources + +- zabbix: https://zabbix.org/wiki/Get_Zabbix +- netdata: https://github.com/netdata/netdata + +## System tools source code + +Source code of system monitoring tools (ps, top, vmstat, etc.) on various +platforms. Useful as a reference when implementing or verifying psutil's +platform-specific code. + +### Linux + +- **procps-ng** (ps, top, free, vmstat, kill, uptime, w, who, pgrep, pmap, + pstree, pwdx, watch): https://gitlab.com/procps-ng/procps +- **sysstat** (iostat, mpstat, pidstat, sar): + https://github.com/sysstat/sysstat +- **iproute2** (ss, ip): https://github.com/iproute2/iproute2 +- **net-tools** (ifconfig, netstat, arp, route): + https://net-tools.sourceforge.io/ +- **util-linux** (mount, findmnt, taskset, ionice, renice, prlimit, lscpu, + lsblk): https://github.com/util-linux/util-linux +- **GNU coreutils** (df, nproc): https://github.com/coreutils/coreutils +- **lsof**: https://github.com/lsof-org/lsof +- **lm-sensors** (sensors): https://github.com/lm-sensors/lm-sensors +- **smem**: https://www.selenic.com/smem/ + +### macOS + +Apple open-source distributions: https://github.com/apple-oss-distributions/ + +- **ps**: https://github.com/apple-oss-distributions/adv_cmds +- **top**: https://github.com/apple-oss-distributions/top +- **sysctl**: https://github.com/apple-oss-distributions/system_cmds +- **netstat, ifconfig**: + https://github.com/apple-oss-distributions/network_cmds +- **lsof**: https://github.com/apple-oss-distributions/lsof + +### FreeBSD + +Repository: https://github.com/freebsd/freebsd-src + +- **ps**: https://github.com/freebsd/freebsd-src/tree/main/bin/ps +- **top**: https://github.com/freebsd/freebsd-src/tree/main/usr.bin/top +- **vmstat**: https://github.com/freebsd/freebsd-src/tree/main/usr.bin/vmstat +- **systat**: https://github.com/freebsd/freebsd-src/tree/main/usr.bin/systat +- **netstat**: https://github.com/freebsd/freebsd-src/tree/main/usr.bin/netstat +- **iostat**: https://github.com/freebsd/freebsd-src/tree/main/usr.sbin/iostat +- **procstat**: + https://github.com/freebsd/freebsd-src/tree/main/usr.bin/procstat +- **ifconfig**: https://github.com/freebsd/freebsd-src/tree/main/sbin/ifconfig +- **cpuset**: https://github.com/freebsd/freebsd-src/tree/main/bin/cpuset +- **pgrep**: https://github.com/freebsd/freebsd-src/tree/main/bin/pkill +- **swapinfo**: + https://github.com/freebsd/freebsd-src/tree/main/usr.sbin/swapinfo + +### NetBSD + +Repository: https://github.com/NetBSD/src + +- **ps**: https://github.com/NetBSD/src/tree/trunk/bin/ps +- **top**: https://github.com/NetBSD/src/tree/trunk/external/bsd/top +- **vmstat**: https://github.com/NetBSD/src/tree/trunk/usr.bin/vmstat +- **systat**: https://github.com/NetBSD/src/tree/trunk/usr.bin/systat +- **netstat**: https://github.com/NetBSD/src/tree/trunk/usr.bin/netstat +- **ifconfig**: https://github.com/NetBSD/src/tree/trunk/sbin/ifconfig +- **pgrep**: https://github.com/NetBSD/src/tree/trunk/bin/pgrep + +### OpenBSD + +Repository: https://github.com/openbsd/src + +- **ps**: https://github.com/openbsd/src/tree/master/bin/ps +- **top**: https://github.com/openbsd/src/tree/master/usr.bin/top +- **vmstat**: https://github.com/openbsd/src/tree/master/usr.bin/vmstat +- **systat**: https://github.com/openbsd/src/tree/master/usr.bin/systat +- **netstat**: https://github.com/openbsd/src/tree/master/usr.bin/netstat +- **ifconfig**: https://github.com/openbsd/src/tree/master/sbin/ifconfig +- **pgrep**: https://github.com/openbsd/src/tree/master/bin/pgrep +- **swapctl**: https://github.com/openbsd/src/tree/master/sbin/swapctl + +### Other tools + +- **zabbix**: https://github.com/zabbix/zabbix/ +- **htop**: https://github.com/htop-dev/htop +- **btop**: https://github.com/aristocratos/btop/ + +## Stats + +- https://pepy.tech/projects/psutil +- https://clickpy.clickhouse.com/dashboard/psutil +- https://pypistats.org/packages/psutil + +## Doc + +### Blog + +Example sites using sphinx ablog: + +- ablog: https://ablog.readthedocs.io/en/stable/blog.html (dogfoods its own + extension) +- SunPy (solar-physics Python library): https://sunpy.org/blog.html +- sgkit (genetics toolkit): https://sgkit-dev.github.io/sgkit/latest/blog.html +- Nuitka (Python-to-C compiler): https://nuitka.net/blog.html +- Executable Books Project (Jupyter Book / MyST): + https://executablebooks.org/en/latest/blog/ +- Adriaan Rol (quantum researcher personal site): https://adriaanrol.com/ +- Jean-Pierre Chauvel (personal tech blog): https://www.chauvel.org/blog/ + +### Nice sites + +- Nice home: https://pnpm.io/ +- https://vite.dev/ diff --git a/docs/Makefile b/docs/Makefile index a69fc329e5..c206dbb2de 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,173 +1,66 @@ # Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -PYTHON = python -SPHINXOPTS = -SPHINXBUILD = $(PYTHON) -m sphinx -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: + +PYTHON = python3 +PYTHONWARNINGS = always,ignore:::sphinx_sitemap,ignore:::matplotlib.projections,ignore:::notfound.extension +SPHINXBUILD = PYTHONWARNINGS="$(PYTHONWARNINGS)" $(PYTHON) -m sphinx +SPHINXAUTOBUILD = PYTHONWARNINGS="$(PYTHONWARNINGS)" sphinx-autobuild +SPHINXOPTS = --fail-on-warning --jobs=auto +BUILDDIR = _build +DOCTREES = $(BUILDDIR)/doctrees +OUTDIR = $(BUILDDIR)/html +DIRHTML = -b dirhtml -d $(DOCTREES) . $(OUTDIR) +# "make autoreload" gets its own dir. Sharing one with the other targets makes +# them delete files from under each other. +LIVEDIR = $(BUILDDIR)/live +LIVE_DIRHTML = -b dirhtml -d $(LIVEDIR)/doctrees . $(LIVEDIR)/html + +# --- build + +clean: ## Remove all build files rm -rf $(BUILDDIR) -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/psutil.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/psutil.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/psutil" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/psutil" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." +html: ## Generate doc in HTML format. Warnings are turned into failures. + $(SPHINXBUILD) $(SPHINXOPTS) $(DIRHTML) + +autoreload: ## Rebuild HTML + live-reload browser on file changes (requires sphinx-autobuild) + $(SPHINXAUTOBUILD) $(SPHINXOPTS) $(LIVE_DIRHTML) + +autoreload-hard: ## Same as above but re-writes all files on every refresh + rm -rf $(LIVEDIR) + $(SPHINXAUTOBUILD) $(SPHINXOPTS) -a $(LIVE_DIRHTML) + +versions: ## Build past doc releases into the current build (needs `html` first) + $(PYTHON) ../scripts/internal/docs/build_versions.py $(OUTDIR) + +# --- checkers + +check-links: ## Check links. Prints only broken URLs / warnings. + $(SPHINXBUILD) $(SPHINXOPTS) --quiet -b linkcheck -d $(DOCTREES) . $(BUILDDIR)/linkcheck + @echo "Link check complete; full report in $(BUILDDIR)/linkcheck/output.txt." + +check-codeautolink: ## Report sphinx-codeautolink resolution failures (non-fatal). + $(SPHINXBUILD) --jobs=auto \ + -D codeautolink_warn_on_failed_resolve=1 \ + -D codeautolink_warn_on_missing_inventory=1 \ + -b dirhtml -d $(BUILDDIR)/codeautolink/doctrees . $(BUILDDIR)/codeautolink/html + +# --- tests + +test: ## Run doc sanity tests. + $(PYTHON) -m pytest test_docs.py $(ARGS) + +test-online-doc: ## Smoke tests against the live docs site. + PSUTIL_DOCS_ONLINE=1 $(PYTHON) -m pytest test_docs_online.py -v + +# --- tools + +blog-post: ## Create a new blog post skeleton + @test -n "$(SLUG)" || { echo "Usage: make blog-post SLUG= [TITLE=] [TAGS=]"; exit 1; } + $(PYTHON) ../scripts/internal/docs/new_blog_post.py "$(SLUG)"$(if $(TITLE), --title "$(TITLE)")$(if $(TAGS), --tags "$(TAGS)") + +refresh-adoption-stats: ## Refresh PyPI/GitHub stats in adoption.rst and README.rst + $(PYTHON) ../scripts/internal/docs/refresh_adoption_stats.py + $(MAKE) -C .. fix-rst + +help: ## Display callable targets. + @awk -F':.*?## ' '/^[a-zA-Z0-9_.-]+:.*?## / {printf "\033[36m%-24s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort diff --git a/docs/README b/docs/README deleted file mode 100644 index 3aaea8a5ba..0000000000 --- a/docs/README +++ /dev/null @@ -1,15 +0,0 @@ -About -===== - -This directory contains the reStructuredText (reST) sources to the psutil -documentation. You don't need to build them yourself, prebuilt versions are -available at https://pythonhosted.org/psutil/. -In case you want, you need to install sphinx first: - - $ pip install sphinx - -Then run: - - $ make html - -You'll then have an HTML version of the doc at _build/html/index.html. \ No newline at end of file diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..11077f42d1 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,64 @@ +# psutil documentation + +This directory is a self-contained Sphinx project that builds the psutil docs +published at https://psutil.io/. It has grown well past a plain API reference +(custom theme, a blog, social cards, its own test suite and deploy pipeline), +so this file is the map. + +## Build & preview + + make html # one-off build into _build/html + make autoreload # live-reload server at 127.0.0.1:8000 + +`make html` turns warnings into errors, same as CI. + +## Layout + +- `*.rst`: the doc sources. `api.rst` is the hand-written API reference; + `index.rst` is the home page. +- `blog/`: blog posts, managed by the ablog extension + comments provided via + giscus. +- `conf.py`: Sphinx config: extensions, the theme, `html_baseurl`, OpenGraph / + sitemap / feed settings. +- `_templates/`: the custom theme, built on Sphinx's `basic` theme (topbar, + sidebar, footer, layout). +- `_static/css/`, `_static/js/`: styles and vanilla JS (no framework). +- `_ext/`: small local Sphinx extensions. +- `_extra/robots.txt`: copied verbatim to the site root. +- `versions.json`: the version selector's menu (see below). + +## Notable choices + +- master is served at the site root (no `/en/`, no `/latest/`); frozen past + releases live under `//`, listed in `versions.json`. +- Built with the `dirhtml` builder, so URLs are extensionless directories + (`psutil.io/faq/`, no `.html`). +- Self-hosted on GitHub Pages under the custom domain psutil.io. +- Fonts, CSS and JS are all self-hosted; no external assets. +- Social cards, sitemap and Atom feed are generated at build time and rooted at + `html_baseurl`. + +## Freezing a doc version + +Add an entry to `versions.json`: + + { "name": "8.0", "url": "/8.0/", "note": "release", "ref": "v8.0.0" } + +Entries with a `ref` are rebuilt from it on every deploy and get an "old +version" banner. Nothing is stored, and tags publish nothing on their own. +Deleting the entry unpublishes it. + +## Tests + +- `test_docs.py`: offline checks on the built HTML (canonical / OG tags, + sitemap, feed, blog metadata, no external assets, ...). Run with `make test`. +- `test_docs_online.py`: smoke tests against the live site (reachability, + http->https, 404 page, metadata). Run with `make test-online-doc` (sets + `PSUTIL_DOCS_ONLINE=1`). + +## Deploy + +`.github/workflows/docs.yml` runs on pushes / PRs that touch `docs/`: lint +(`make lint-rst`) -> offline tests -> build -> build past versions -> deploy to +GitHub Pages -> live-site tests. Deploy and the live tests run only on push to +master, never on PRs. diff --git a/docs/_ext/ablog_extras.py b/docs/_ext/ablog_extras.py new file mode 100644 index 0000000000..43a20bed0e --- /dev/null +++ b/docs/_ext/ablog_extras.py @@ -0,0 +1,49 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Ablog tweaks.""" + +import pathlib +import xml.etree.ElementTree as ET + +SITEMAP_NS = "http://www.sitemaps.org/schemas/sitemap/0.9" + + +def merge_ablog_posts(app, env, docnames, other): + if hasattr(other, "ablog_posts"): + if not hasattr(env, "ablog_posts"): + env.ablog_posts = {} + env.ablog_posts.update(other.ablog_posts) + + +def dedupe_sitemap(app, exception): + # ablog re-renders the blog index on top of blog.rst, so + # sphinx-sitemap sees that page twice and lists its URL twice. + if exception: + return + path = pathlib.Path(app.outdir) / app.config.sitemap_filename + if not path.is_file(): + return + ET.register_namespace("", SITEMAP_NS) + tree = ET.parse(path) + root = tree.getroot() + seen = set() + for url in list(root): + loc = url.findtext(f"{{{SITEMAP_NS}}}loc") + if loc in seen: + root.remove(url) + else: + seen.add(loc) + tree.write(path, encoding="utf-8", xml_declaration=True) + + +def setup(app): + app.connect("env-merge-info", merge_ablog_posts) + # priority > 500 so this runs after sphinx-sitemap writes the file. + app.connect("build-finished", dedupe_sitemap, priority=600) + # Ablog stores posts on the build environment but doesn't merge + # them back when Sphinx builds in parallel, so worker results get + # dropped. See https://github.com/sunpy/ablog/pull/330. + app.extensions["ablog"].parallel_read_safe = True + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_ext/availability.py b/docs/_ext/availability.py new file mode 100644 index 0000000000..99e7d8705c --- /dev/null +++ b/docs/_ext/availability.py @@ -0,0 +1,100 @@ +# noqa: CPY001 + +# Slightly adapted from CPython's: +# https://github.com/python/cpython/blob/main/Doc/tools/extensions/availability.py +# Copyright (c) PSF +# Licensed under the Python Software Foundation License Version 2. + +"""Support for `.. availability:: …` directive, to document platform +availability. +""" + +from docutils import nodes +from sphinx.locale import _ as sphinx_gettext +from sphinx.util import logging +from sphinx.util.docutils import SphinxDirective + +logger = logging.getLogger(__name__) + +_PLATFORMS = frozenset({ + "AIX", + "BSD", + "FreeBSD", + "Linux", + "Linux with glibc", + "macOS", + "NetBSD", + "OpenBSD", + "POSIX", + "SunOS", + "UNIX", + "Windows", +}) + +_LIBC = frozenset({ + "glibc", + "musl", +}) + +KNOWN_PLATFORMS = _PLATFORMS | _LIBC + + +class Availability(SphinxDirective): + has_content = True + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = True + + def run(self): + title = sphinx_gettext("Availability") + sep = nodes.Text(" ") + parsed, msgs = self.state.inline_text(self.arguments[0], self.lineno) + pnode = nodes.paragraph( + title, "", nodes.emphasis(title, title + ":"), sep, *parsed, *msgs + ) + self.set_source_info(pnode) + cnode = nodes.container("", pnode, classes=["availability"]) + self.set_source_info(cnode) + if self.content: + self.state.nested_parse(self.content, self.content_offset, cnode) + self.parse_platforms() + + return [cnode] + + def parse_platforms(self): + """Parse platform information from arguments + + Arguments is a comma-separated string of platforms. A platform may + be prefixed with "not " to indicate that a feature is not available. + Example: + .. availability:: Windows, Linux >= 4.2, not glibc + """ + platforms = {} + for arg in self.arguments[0].rstrip(".").split(","): + arg = arg.strip() + platform, _, version = arg.partition(" >= ") + if platform.startswith("not "): + version = False + platform = platform.removeprefix("not ") + elif not version: + version = True + platforms[platform] = version + + unknown = set(platforms).difference(KNOWN_PLATFORMS) + if unknown: + logger.warning( + "Unknown platform%s or syntax '%s' in '.. availability:: %s', " + "see %s:KNOWN_PLATFORMS for a set of known platforms.", + "s" if len(platforms) != 1 else "", + " ".join(sorted(unknown)), + self.arguments[0], + __file__, + location=self.get_location(), + ) + + return platforms + + +def setup(app): + app.add_directive("availability", Availability) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_ext/changelog_anchors.py b/docs/_ext/changelog_anchors.py new file mode 100644 index 0000000000..2598a3cc6e --- /dev/null +++ b/docs/_ext/changelog_anchors.py @@ -0,0 +1,46 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Sphinx extension for adding anchors to each section in changelog.rst. + +This script gets called on `make html`, and adds numeric anchors for +version titles (e.g., `7.2.3 — 2026-02-08` -> #723). It also registers +them as Sphinx labels so that :ref:`722` cross-references resolve +correctly. +""" + +import re + +from docutils import nodes + +VERSION_RE = re.compile(r"^(\d+\.\d+\.\d+)") + + +def add_version_anchors(app, doctree): + docname = app.env.docname + if docname != "changelog": + return + + labels = app.env.domaindata.setdefault('std', {}).setdefault('labels', {}) + anonlabels = app.env.domaindata['std'].setdefault('anonlabels', {}) + + for node in doctree.findall(nodes.section): + title = node.next_node(nodes.title) + if not title: + continue + + text = title.astext() + m = VERSION_RE.match(text) + if m: + anchor = m.group(1).replace(".", "") + if anchor not in node["ids"]: + node["ids"].insert(0, anchor) + if anchor not in labels: + labels[anchor] = ("changelog", anchor, text) + anonlabels[anchor] = ("changelog", anchor) + + +def setup(app): + app.connect("doctree-read", add_version_anchors) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_ext/check_python_syntax.py b/docs/_ext/check_python_syntax.py new file mode 100644 index 0000000000..1758535b48 --- /dev/null +++ b/docs/_ext/check_python_syntax.py @@ -0,0 +1,45 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Sphinx extension that checks the Python syntax of code blocks in the +documentation. This script gets called on `make html`. +""" + +import ast + +import docutils.nodes +import sphinx.errors + + +def check_python_blocks(app, doctree, docname): + path = app.env.doc2path(docname) + + for node in doctree.findall(docutils.nodes.literal_block): + lang = node.get("language") + if lang not in {"python", "py"}: + continue + + code = node.astext() + + # skip empty blocks + if not code.strip(): + continue + + # skip REPL examples containing >>> + if ">>>" in code: + continue + + try: + ast.parse(code, feature_version=(3, 8)) + except SyntaxError as err: + lineno = node.line or "?" + msg = ( + f"invalid Python syntax in {path}:{lineno}:\n\n{code}\n\n{err}" + ) + raise sphinx.errors.SphinxError(msg) from None + + +def setup(app): + app.connect("doctree-resolved", check_python_blocks) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_ext/field_role.py b/docs/_ext/field_role.py new file mode 100644 index 0000000000..c1b475a63b --- /dev/null +++ b/docs/_ext/field_role.py @@ -0,0 +1,22 @@ +# Copyright (c) 2009, Giampaolo Rodola. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Sphinx extension providing the :field:`name` role for marking named +tuple fields in the API doc. +""" + +from docutils import nodes + + +def field_role( + name, rawtext, text, lineno, inliner, options=None, content=None +): + """Render :field:`name` as inline code (monospace bold).""" + node = nodes.literal(rawtext, text, classes=["ntuple-field"]) + return [node], [] + + +def setup(app): + app.add_role("field", field_role) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_ext/genindex_filter.py b/docs/_ext/genindex_filter.py new file mode 100644 index 0000000000..6a84797238 --- /dev/null +++ b/docs/_ext/genindex_filter.py @@ -0,0 +1,57 @@ +# Copyright (c) 2009, Giampaolo Rodola. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Post-process genindex.html to clean up index entry labels. + +Transforms: + NAME (qualifier) + +Into: + NAME (qualifier) + +Also shortens verbose Sphinx qualifiers: + (in module psutil) → (function) if NAME ends with "()", else (constant) + (class in psutil) → (class) + (exception in psutil) → (exception) +""" + +import pathlib +import re + +# Matches NAME (qualifier) where NAME does not start +# with "(" — leaving pure-paren sub-entries like "(psutil.Foo method)" alone. +REGEX = re.compile(r'([^(<][^<]*?) (\([^)]+\))') + + +def replace(m): + href, name, qualifier = m.group(1), m.group(2), m.group(3) + if qualifier == "(in module psutil)": + qualifier = "(function)" if name.endswith("()") else "(constant)" + elif qualifier == "(class in psutil)": + qualifier = "(class)" + elif qualifier == "(exception in psutil)": + qualifier = "(exception)" + return ( + f'{name} {qualifier}' + ) + + +def on_build_finished(app, exception): + # format (not name) so this also runs under dirhtml, which writes + # genindex/index.html rather than genindex.html. + if exception or app.builder.format != "html": + return + genindex = pathlib.Path(app.builder.get_outfilename("genindex")) + if not genindex.exists(): + return + original = genindex.read_text(encoding="utf-8") + processed = REGEX.sub(replace, original) + if processed != original: + genindex.write_text(processed, encoding="utf-8") + + +def setup(app): + app.connect("build-finished", on_build_finished) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_ext/giscus.py b/docs/_ext/giscus.py new file mode 100644 index 0000000000..241224bddf --- /dev/null +++ b/docs/_ext/giscus.py @@ -0,0 +1,31 @@ +# Copyright (c) 2009, Giampaolo Rodola. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Feed the giscus config and a per-page "is this a blog post" flag to +the HTML templates, so _templates/comments.html can render the comment +widget on post pages only. +""" + +CONFIG_VALUES = ( + "giscus_repo", + "giscus_repo_id", + "giscus_category", + "giscus_category_id", +) + + +def add_giscus_context(app, pagename, templatename, context, doctree): + is_post = pagename in getattr(app.env, "ablog_posts", {}) + # A bare ":no_comments:" field at the top of a post opts it out. + meta = app.env.metadata.get(pagename, {}) + context["is_blog_post"] = is_post and "no_comments" not in meta + for name in CONFIG_VALUES: + context[name] = getattr(app.config, name) + + +def setup(app): + for name in CONFIG_VALUES: + app.add_config_value(name, "", "html") + app.connect("html-page-context", add_giscus_context) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_ext/glossary_toc.py b/docs/_ext/glossary_toc.py new file mode 100644 index 0000000000..78f268db96 --- /dev/null +++ b/docs/_ext/glossary_toc.py @@ -0,0 +1,27 @@ +# Copyright (c) 2009, Giampaolo Rodola. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Inject glossary terms into the Jinja context to enable rendering of +the right-side TOC on /glossary. +""" + +from docutils import nodes + + +def on_html_page_context(app, page_name, template_name, context, doctree): + if page_name != "glossary": + return + glossary_doctree = app.env.get_doctree("glossary") + terms = [] + for term in glossary_doctree.findall(nodes.term): + ids = term.get("ids") or [] + if ids: + terms.append({"id": ids[0], "text": term.astext()}) + terms.sort(key=lambda t: t["text"].lower()) + context["glossary_terms"] = terms + + +def setup(app): + app.connect("html-page-context", on_html_page_context) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_ext/label_role.py b/docs/_ext/label_role.py new file mode 100644 index 0000000000..7b536d2c74 --- /dev/null +++ b/docs/_ext/label_role.py @@ -0,0 +1,30 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Sphinx role providing the changelog labels, e.g. :label:`critical`.""" + +from docutils import nodes + +LABELS = ("breaking", "critical", "build-fail", "memleak") + + +def label_role( + name, rawtext, text, lineno, inliner, options=None, content=None +): + label = text.strip().lower() + if label not in LABELS: + msg = inliner.reporter.error( + f"unknown label {text!r}, expected one of {', '.join(LABELS)}", + line=lineno, + ) + return [inliner.problematic(rawtext, rawtext, msg)], [msg] + node = nodes.inline( + rawtext, label, classes=["cl-label", f"cl-label-{label}"] + ) + return [node], [] + + +def setup(app): + app.add_role("label", label_role) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_ext/notfound_extras.py b/docs/_ext/notfound_extras.py new file mode 100644 index 0000000000..936d3f24cd --- /dev/null +++ b/docs/_ext/notfound_extras.py @@ -0,0 +1,56 @@ +# Copyright (c) 2009, Giampaolo Rodola. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Make the custom 404 page work under dirhtml + static hosting. + +Two gaps sphinx-notfound-page / dirhtml leave, both about URLs on the +404 page: + +- notfound absolutizes the 404's chrome and images but leaves body + cross-references (:doc:, :class:) relative. Those break when the 404 + is served from a deep path, so absolutize them too, reusing + notfound's own helper. +- dirhtml writes the 404 as ``404/index.html``. GitHub Pages and + Starlette (sphinx-autobuild) serve the custom 404 from ``/404.html`` + at the root, so materialize it there. +""" + +import shutil +from pathlib import Path + +from docutils import nodes +from notfound.utils import replace_uris + + +def absolutize_404_links(app, doctree, docname): + if docname == app.config.notfound_pagename: + replace_uris(app, doctree, nodes.reference, "refuri") + + +def absolutize_404_content_root(app, pagename, templatename, context, doctree): + # data-content_root is relative; JS (search, highlight) uses it to + # build URLs. On the 404, served from any depth, it must be + # absolute or those URLs resolve against the wrong base. + if pagename == app.config.notfound_pagename: + context["content_root"] = app.config.notfound_urls_prefix + + +def write_root_404(app, exception): + if exception is not None: + return + name = app.config.notfound_pagename + src = Path(app.outdir) / name / "index.html" + if src.is_file(): + shutil.copyfile(src, Path(app.outdir) / f"{name}.html") + + +def setup(app): + app.connect("doctree-resolved", absolutize_404_links) + app.connect("html-page-context", absolutize_404_content_root) + app.connect("build-finished", write_root_404) + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/docs/_ext/opengraph_override.py b/docs/_ext/opengraph_override.py new file mode 100644 index 0000000000..74eac575db --- /dev/null +++ b/docs/_ext/opengraph_override.py @@ -0,0 +1,151 @@ +# Copyright (c) 2009, Giampaolo Rodola. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Bridge ablog and sphinxext-opengraph for social-card metadata.""" + +import html +import re + +import sphinxext.opengraph + +BLOG_DESCRIPTION = "Psutil blog: releases, deep dives, war stories" +HOME_OG_TITLE = "psutil: Process and System Utilities for Python" +DESCRIPTION_OVERRIDE = {} + +STATIC_DESCRIPTIONS = { + "index": ( + "psutil is a cross-platform Python library for retrieving " + "information on running processes and system utilization: CPU, " + "memory, disks, network and sensors." + ), + "api": "psutil full API reference.", + "install": "How to install psutil.", +} + +VIEWPORT_RE = re.compile(r'\s*]*>', re.IGNORECASE) +OG_TITLE_RE = re.compile(r'( desc_len: + text = text[: desc_len - 3] + "..." + return text + + +def capture_post_summary(app, pagename, templatename, context, doctree): + # Get post's summary (the body of the .. post:: directive) and use + # it in the og preview. + DESCRIPTION_OVERRIDE.pop("current", None) + static = STATIC_DESCRIPTIONS.get(pagename) + if static: + DESCRIPTION_OVERRIDE["current"] = static + return + posts = (getattr(app.env, "ablog_posts", {}) or {}).get(pagename) or [] + if not posts: + return + excerpt_nodes = posts[0].get("excerpt") + if not excerpt_nodes: + return + text = " ".join(n.astext() for n in excerpt_nodes if hasattr(n, "astext")) + text = " ".join(text.split()) + if text: + DESCRIPTION_OVERRIDE["current"] = text + + +def set_og_type_article(app, pagename, templatename, context, doctree): + # sphinxext-opengraph hardcodes og:type="website" globally. Override + # it to "article" for blog posts so social platforms classify them. + if pagename in getattr(app.env, "ablog_posts", {}): + meta = context.get("meta") + if meta is None: + meta = {} + context["meta"] = meta + meta["og:type"] = "article" + + +def blog_index_meta(app, pagename): + """Title and description for an ablog-generated index page.""" + blog_path = app.config.blog_path + if pagename == blog_path: + return app.config.project + " blog", BLOG_DESCRIPTION + m = re.fullmatch(re.escape(blog_path) + r"/(\d{4})", pagename) + if m: + year = m.group(1) + return ( + f"{app.config.project} blog: {year}", + f"psutil blog posts published in {year}.", + ) + return None + + +def emit_blog_index_meta(app, pagename, templatename, context, doctree): + # ablog renders the blog index and the year archives without a + # doctree, so sphinxext-opengraph skips them: no description, no + # og:* tags at all. + meta = blog_index_meta(app, pagename) + if meta is None: + return + og_title, description = meta + project = app.config.project + base = app.config.html_baseurl.rstrip("/") + "/" + fields = [ + ("name", "description", description), + ("property", "og:title", og_title), + ("property", "og:type", "website"), + # Builder-derived: dirhtml serves this as blog/, not blog.html. + ("property", "og:url", base + app.builder.get_target_uri(pagename)), + ("property", "og:site_name", project), + ("property", "og:description", description), + ("property", "og:image", base + "_static/images/logo-psutil.png"), + ( + "property", + "og:image:alt", + ( + "psutil blog: articles on processes, system monitoring " + "and psutil development" + ), + ), + ("name", "twitter:card", "summary"), + ] + tags = "\n".join( + f'' for attr, key, val in fields + ) + if pagename == app.config.blog_path: + context["feed_title"] = app.config.blog_title or "Blog" + context["metatags"] = context.get("metatags", "") + tags + "\n" + + +def finalize_head(app, pagename, templatename, context, doctree): + tags = context.get("metatags") + if not tags: + return + deduped = VIEWPORT_RE.sub("", tags) + if pagename == app.config.master_doc: + deduped = OG_TITLE_RE.sub( + r"\g<1>" + html.escape(HOME_OG_TITLE, quote=True) + r"\g<2>", + deduped, + ) + if deduped != tags: + context["metatags"] = deduped + + +def setup(app): + sphinxext.opengraph.get_description = patched_get_description + + # priority < 500 ensures we run before sphinxext-opengraph. + app.connect("html-page-context", capture_post_summary, priority=300) + app.connect("html-page-context", set_og_type_article, priority=400) + app.connect("html-page-context", emit_blog_index_meta) + app.connect("html-page-context", finalize_head, priority=600) + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/docs/_ext/post_banner.py b/docs/_ext/post_banner.py new file mode 100644 index 0000000000..8c907b77ae --- /dev/null +++ b/docs/_ext/post_banner.py @@ -0,0 +1,163 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Insert a metadata banner (date + author + tags + reading time) +right below the H1 of each blog post. + +Ablog's sidebar exposes post metadata, but nothing appears near the +title on the post page itself. This extension walks post doctrees +and inserts a small container after the first section's title. +""" + +import math + +from ablog.blog import Blog +from docutils import nodes + +# Conservative words-per-minute estimate for technical prose. Code +# blocks are excluded from the word count. +WPM = 200 +SKIP_NODES = ( + nodes.literal_block, + nodes.doctest_block, + nodes.comment, + nodes.raw, + nodes.system_message, +) + + +def count_words(node): + if isinstance(node, SKIP_NODES): + return 0 + if isinstance(node, nodes.Text): + return len(node.astext().split()) + return sum(count_words(c) for c in node.children) + + +def reading_minutes(doctree): + return max(1, math.ceil(count_words(doctree) / WPM)) + + +def tag_ref(app, blog, docname, label, text, classes=None): + """Return a nodes.reference from `docname` to the tag page for + `label`. + """ + coll = blog.tags[label] + # Let the builder compute this: dirhtml serves each page as a + # directory, so a hand-rolled relpath + ".html" lands a level off + # and points at a file that doesn't exist. + target = app.builder.get_relative_uri(docname, coll.docname) + return nodes.reference( + text, text, refuri=target, internal=True, classes=classes or [] + ) + + +def featured_inline(app, blog, post, docname): + tags = [str(t) for t in post.get("tags") or []] + if "featured" not in tags: + return None + # Reference must live inside a TextElement (sphinx html5 asserts + # this). Wrap in an inline like tags_inline does. + container = nodes.inline() + container += tag_ref( + app, blog, docname, "featured", "Featured", ["post-meta-featured"] + ) + return container + + +def author_inline(post): + authors = post.get("author") or [] + if not authors: + return None + text = ", ".join(str(a) for a in authors) + return nodes.inline(text, text, classes=["post-meta-author"]) + + +def date_inline(post): + date = post.get("date") + if not date: + return None + text = date.strftime("%b %d, %Y") + return nodes.inline(text, text, classes=["post-meta-date"]) + + +def readtime_inline(doctree): + text = f"{reading_minutes(doctree)} min read" + return nodes.inline(text, text, classes=["post-meta-readtime"]) + + +def tags_inline(app, blog, post, docname): + tags = [t for t in post.get("tags") or [] if str(t) != "featured"] + if not tags: + return None + container = nodes.inline(classes=["post-meta-tags"]) + for label in sorted(tags, key=str): + container += tag_ref(app, blog, docname, label, str(blog.tags[label])) + return container + + +def find_title_position(doctree): + """Return (section, title_index) for the first section with a + title, or (None, None) if none exists. + """ + section = next(iter(doctree.findall(nodes.section)), None) + if section is None: + return None, None + idx = next( + ( + i + for i, c in enumerate(section.children) + if isinstance(c, nodes.title) + ), + None, + ) + return (section, idx) if idx is not None else (None, None) + + +class post_banner(nodes.container, nodes.Invisible): + """Container subclass marked as docutils Invisible. The HTML + writer still renders it as a
(via the visit/depart + functions registered below), but sphinxext-opengraph's + description parser skips isinstance(node, nodes.Invisible) + nodes, so the banner text does not leak into og:description. + """ + + +def visit_post_banner(self, node): + self.visit_container(node) + + +def depart_post_banner(self, node): + self.depart_container(node) + + +def insert_banner(app, doctree, docname): + posts = getattr(app.env, "ablog_posts", {}).get(docname) + if not posts: + return + post = posts[0] + + section, title_idx = find_title_position(doctree) + if section is None: + return + + blog = Blog(app) + banner = post_banner(classes=["post-meta-banner"]) + for child in ( + featured_inline(app, blog, post, docname), + author_inline(post), + date_inline(post), + readtime_inline(doctree), + tags_inline(app, blog, post, docname), + ): + if child is not None: + banner += child + + section.insert(title_idx + 1, banner) + + +def setup(app): + app.add_node(post_banner, html=(visit_post_banner, depart_post_banner)) + app.connect("doctree-resolved", insert_banner) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_ext/proc_role.py b/docs/_ext/proc_role.py new file mode 100644 index 0000000000..1e6af386c6 --- /dev/null +++ b/docs/_ext/proc_role.py @@ -0,0 +1,32 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Sphinx role linking `/proc/` paths to Debian's man-pages. +E.g. :proc:`/proc/meminfo`, :proc:`/proc/pid/statm`. +""" + +from docutils import nodes + +URL = "https://manpages.debian.org/{}(5)" + + +def proc_role( + name, rawtext, text, lineno, inliner, options=None, content=None +): + path = text.strip() + # /proc/[pid]/stat -> proc_pid_stat + parts = [] + for seg in path.lstrip("/").split("/"): + s = seg.strip("[]").lower() + parts.append("pid" if s == "pid" else s) + page = "_".join(parts) + url = URL.format(page) + ref = nodes.reference(rawtext, "", refuri=url, classes=["proc"]) + ref += nodes.literal(text=path) + return [ref], [] + + +def setup(app): + app.add_role("proc", proc_role) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_ext/substitutions.py b/docs/_ext/substitutions.py new file mode 100644 index 0000000000..2034645a32 --- /dev/null +++ b/docs/_ext/substitutions.py @@ -0,0 +1,39 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Expand {{name}} tokens in the .rst sources. + +For values that would otherwise be hardcoded and go stale. Docutils +substitutions (|name|) can't be used: they don't expand inside +`raw:: html` blocks, and the home page stats live in one. This runs +on the raw text, before parsing, so tokens work anywhere. +""" + +import datetime + +# Date of the first commit. A constant so the build doesn't shell out +# to git (and doesn't depend on a full clone); test_docs.py checks it +# against the real history. +FIRST_COMMIT = datetime.date(2008, 4, 22) + + +def years_in_development(today=None): + today = today or datetime.date.today() + years = today.year - FIRST_COMMIT.year + if (today.month, today.day) < (FIRST_COMMIT.month, FIRST_COMMIT.day): + years -= 1 + return years + + +def substitute(app, docname, source): + values = {"years_in_development": years_in_development()} + text = source[0] + for name, value in values.items(): + text = text.replace("{{" + name + "}}", str(value)) + source[0] = text + + +def setup(app): + app.connect("source-read", substitute) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_extra/robots.txt b/docs/_extra/robots.txt new file mode 100644 index 0000000000..47df22042a --- /dev/null +++ b/docs/_extra/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://psutil.io/sitemap.xml diff --git a/docs/_sponsors.html b/docs/_sponsors.html new file mode 100644 index 0000000000..a6c938ea8b --- /dev/null +++ b/docs/_sponsors.html @@ -0,0 +1,65 @@ + + + + + + + + + diff --git a/docs/_static/copybutton.js b/docs/_static/copybutton.js deleted file mode 100644 index 5d82c672be..0000000000 --- a/docs/_static/copybutton.js +++ /dev/null @@ -1,57 +0,0 @@ -$(document).ready(function() { - /* Add a [>>>] button on the top-right corner of code samples to hide - * the >>> and ... prompts and the output and thus make the code - * copyable. */ - var div = $('.highlight-python .highlight,' + - '.highlight-python3 .highlight') - var pre = div.find('pre'); - - // get the styles from the current theme - pre.parent().parent().css('position', 'relative'); - var hide_text = 'Hide the prompts and output'; - var show_text = 'Show the prompts and output'; - var border_width = pre.css('border-top-width'); - var border_style = pre.css('border-top-style'); - var border_color = pre.css('border-top-color'); - var button_styles = { - 'cursor':'pointer', 'position': 'absolute', 'top': '0', 'right': '0', - 'border-color': border_color, 'border-style': border_style, - 'border-width': border_width, 'color': border_color, 'text-size': '75%', - 'font-family': 'monospace', 'padding-left': '0.2em', 'padding-right': '0.2em', - 'border-radius': '0 3px 0 0' - } - - // create and add the button to all the code blocks that contain >>> - div.each(function(index) { - var jthis = $(this); - if (jthis.find('.gp').length > 0) { - var button = $('>>>'); - button.css(button_styles) - button.attr('title', hide_text); - jthis.prepend(button); - } - // tracebacks (.gt) contain bare text elements that need to be - // wrapped in a span to work with .nextUntil() (see later) - jthis.find('pre:has(.gt)').contents().filter(function() { - return ((this.nodeType == 3) && (this.data.trim().length > 0)); - }).wrap(''); - }); - - // define the behavior of the button when it's clicked - $('.copybutton').toggle( - function() { - var button = $(this); - button.parent().find('.go, .gp, .gt').hide(); - button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden'); - button.css('text-decoration', 'line-through'); - button.attr('title', show_text); - }, - function() { - var button = $(this); - button.parent().find('.go, .gp, .gt').show(); - button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible'); - button.css('text-decoration', 'none'); - button.attr('title', hide_text); - }); -}); - diff --git a/docs/_static/css/admonitions.css b/docs/_static/css/admonitions.css new file mode 100644 index 0000000000..a98d3510bd --- /dev/null +++ b/docs/_static/css/admonitions.css @@ -0,0 +1,135 @@ +/* ---- Shared structure (defaults to Note) ---------------------------- */ + +div.admonition { + position: relative; + margin: 10px 0; + padding: var(--adm-padding-y) 10px var(--adm-padding-y) 36px; + background-color: var(--adm-note-bg); + border: 1px solid var(--adm-note-border); + border-left: 4px solid var(--adm-note-accent-border); + border-radius: 4px; +} + +div.admonition + div.admonition { + margin-top: 16px; +} + +div.admonition p { + margin-bottom: var(--adm-padding-y); +} + +/* ---- Per-type color overrides --------------------------------------- */ + +div.warning { + background-color: var(--adm-warning-bg); + border-color: var(--adm-warning-border); + border-left-color: var(--adm-warning-border); +} + +div.tip { + background-color: var(--adm-tip-bg); + border-color: var(--adm-tip-border); + border-left-color: var(--adm-tip-border); +} + +div.important { + background-color: var(--adm-important-bg); + border-color: var(--adm-important-border); + border-left-color: var(--adm-important-border); +} + +div.seealso { + background-color: var(--adm-seealso-bg); + border-color: var(--adm-seealso-border); + border-left-color: var(--adm-seealso-accent-border); +} + +div.seealso a.reference { + color: var(--adm-seealso-link); + text-decoration: underline; + text-underline-offset: 2px; +} + +div.seealso a.reference:has(code) { + text-decoration: none; +} + +div.seealso a.reference:hover { + color: var(--adm-seealso-hover); +} + +div.admonition a { + text-decoration: none; +} + +/* ---- Title (inline with the first paragraph of content) ------------ */ + +p.admonition-title, +p.admonition-title + p { + display: inline; + margin: 0; + padding: 0; +} + +p.admonition-title { + background: none; + color: var(--adm-title); + font-weight: 700; + margin-right: 0.4em; +} + +p.admonition-title::after { + content: ":"; +} + +p.admonition-title::before { + content: "\f129"; + position: absolute; + left: 10px; + top: calc(var(--adm-padding-y) + 3px); + display: flex; + align-items: center; + justify-content: center; + width: 16px; + height: 16px; + border: 1.5px solid var(--adm-note-accent-border); + border-radius: 50%; + font-family: "Font Awesome 7 Free"; + font-weight: 900; + font-size: 10px; + line-height: 1; + color: var(--adm-note-accent-border); +} + +div.tip p.admonition-title::before, +div.warning p.admonition-title::before, +div.important p.admonition-title::before, +div.seealso p.admonition-title::before { + top: var(--adm-padding-y); + width: auto; + height: auto; + border: 0; + border-radius: 0; + font-size: 15px; + line-height: inherit; +} + +div.tip p.admonition-title::before { + content: "\f0eb"; + color: var(--adm-tip-icon); +} + +div.warning p.admonition-title::before { + content: "\f071"; + color: var(--adm-warning-icon); +} + +div.important p.admonition-title::before { + content: "\f06a"; + color: var(--adm-important-icon); +} + +div.seealso p.admonition-title::before { + content: "\f0c1"; + color: var(--adm-seealso-accent-border); +} diff --git a/docs/_static/css/api-signatures.css b/docs/_static/css/api-signatures.css new file mode 100644 index 0000000000..e411f3a129 --- /dev/null +++ b/docs/_static/css/api-signatures.css @@ -0,0 +1,128 @@ +/* + * psutil-sphinx-theme: API function / class signatures. + */ + +/* ---- Signature container -------------------------------------------- */ + +.article dl:not(.docutils):not(.simple) { + margin: 0 0 0 0; +} + +.article dl.py + dl.py > dt { + margin-top: 10px; +} + +/* Collapse constants without a description body. */ +.article dl.py:not(:has(> dd p)) + dl.py > dt { + margin-top: 0px; +} + +.article dl.py > dd:not(:has(p)) { + margin: 0; + padding: 0; +} + +.article dl:not(.docutils):not(.glossary) > dt { + background: var(--func-sig-bg); + color: var(--func-sig-text); + padding: 1px 9px; + border-left: 3px solid var(--func-sig-border); + border-radius: 0; + font-family: var(--font-body); + font-size: 17px; + margin-top: 0; + margin-bottom: 10px; + font-weight: 500; + display: block; + line-height: 1.5; + position: relative; +} + +.article dl:not(.docutils):not(.glossary) > dt a.headerlink { + margin-left: 0.4rem; + padding: 0; +} + +/* Function name + default values (e.g. "False" / "None"): bold. */ +.article dl:not(.docutils) > dt .descname, +.article dl:not(.docutils) > dt .sig-name, +.article dl:not(.docutils) > dt .default_value { + color: var(--func-sig-name); + font-weight: 700; +} + +/* Reset to muted/normal: psutil. prefix, parens, param names, + operators, properties. Default values keep 700 from the dt. */ +.article dl:not(.docutils) > dt .sig-paren, +.article dl:not(.docutils) > dt .sig-prename, +.article dl:not(.docutils) > dt .descclassname, +.article dl:not(.docutils) > dt .n, +.article dl:not(.docutils) > dt .o, +.article dl:not(.docutils) > dt .property { + color: var(--func-sig-param); + font-weight: 400; +} + +.sig-paren { + padding-left: 2px; + padding-right: 2px; +} + +/* "[source]" link to the right of the signature. */ +.viewcode-link { + color: var(--func-sig-border); + font-size: 0.8em; + font-style: italic; + font-weight: 600; + text-decoration: none; + margin-left: 1.5em; + vertical-align: middle; + position: relative; + top: 5px; +} + +.viewcode-link:hover { + text-decoration: underline; +} + +/* Nested
body for the doc paragraph. */ +.article dl:not(.docutils):not(.glossary) > dd { + margin: 0.6em 0 1em 0; + padding: 0 0 0 1em; +} + +/* ---- Glossary opt-out ---------------------------------------------- */ + +.article dl.glossary > dt { + background: transparent; + border: none; + padding: 0; + font-size: 1.1em; + font-weight: 600; + color: var(--headings); + margin: 1.6em 0 0.3em 0; +} + +.article dl.glossary > dd { + margin: 0 0 1em 1.5em; + padding: 0; + background: transparent; + border: none; +} + +/* ---- Inline ntuple field code (e.g. `user`, `system`) -------------- */ + +code.ntuple-field, +.ntuple-field { + color: var(--ntuple-color); + font-weight: 700; +} + +/* Compress constants */ +.article dl.py.data > dt { + margin-bottom: 2px; +} + +.article dl.py.data + dl.py.data > dt { + margin-top: 0; +} diff --git a/docs/_static/css/banner.css b/docs/_static/css/banner.css new file mode 100644 index 0000000000..c8f9467d83 --- /dev/null +++ b/docs/_static/css/banner.css @@ -0,0 +1,105 @@ +/* Full-width notice above the topbar (see _templates/banner.html). + Archived releases inject the same markup from build_versions.py. */ + +.site-banner { + display: flex; + align-items: center; + justify-content: center; + gap: 10px; + padding: 9px 44px 9px 14px; + position: relative; + background: #fff8e1; + border-bottom: 1px solid #f0d98c; + color: #6b5618; + font-size: 0.88rem; + line-height: 1.45; + text-align: center; +} + +html.site-banner-dismissed .site-banner { + display: none; +} + +/* Archived releases are built from an old tag and have no + .header-stack; there the banner pins itself. */ +body > .site-banner { + position: sticky; + top: 0; + z-index: 1000; +} + +/* Sensible offset before (or without) js/banner.js, which replaces it + with the measured height. */ +html:has(.site-banner) { + --header-height: calc(var(--topbar-height) + 40px); +} + +html.site-banner-dismissed { + --header-height: var(--topbar-height); +} + +.site-banner-close::before { + content: "\00d7"; + font-size: 19px; + line-height: 1; +} + +.site-banner-close { + position: absolute; + right: 10px; + display: inline-flex; + align-items: center; + justify-content: center; + width: 26px; + height: 26px; + padding: 0; + border: 0; + border-radius: 50%; + background: transparent; + color: inherit; + opacity: 0.7; + cursor: pointer; +} + +.site-banner-close:hover { + background: rgba(0, 0, 0, 0.07); + opacity: 1; +} + +.site-banner-close:focus-visible { + outline: 2px solid currentColor; + outline-offset: 1px; +} + +[data-theme="dark"] .site-banner-close:hover { + background: rgba(255, 255, 255, 0.12); +} + +/* Paired with :visited so the theme's visited-link rule can't leak + through and recolor it mid-sentence. */ +.site-banner a, +.site-banner a:visited { + color: #8a6d1f; + font-weight: 600; + text-decoration: underline; + text-underline-offset: 2px; +} + +.site-banner a:hover { + color: #5c470f; +} + +[data-theme="dark"] .site-banner { + background: #2b2415; + border-bottom-color: #4a3c1d; + color: #ddcb98; +} + +[data-theme="dark"] .site-banner a, +[data-theme="dark"] .site-banner a:visited { + color: #f0d98c; +} + +[data-theme="dark"] .site-banner a:hover { + color: #fbecc0; +} diff --git a/docs/_static/css/base.css b/docs/_static/css/base.css new file mode 100644 index 0000000000..4bbc31131c --- /dev/null +++ b/docs/_static/css/base.css @@ -0,0 +1,415 @@ +/* + * psutil-sphinx-theme: design tokens. + * + * All shared variables (colors, fonts, layout dimensions) live here + * so per-component CSS files (layout.css, topbar.css, *.css) consume + * vars instead of hardcoded values. Loaded first so the cascade has + * the tokens available before any rule references them. + */ + +:root { + color-scheme: light; + + /* ================================================================== */ + /* Layout */ + /* ================================================================== */ + + --topbar-height: 44px; + --header-height: var(--topbar-height); + --left-sidebar-width: 260px; + --right-sidebar-width: 230px; + --content-max-width: 1240px; + --layout-width: calc(var(--left-sidebar-width) + var(--content-max-width)); + + /* ================================================================== */ + /* Colors */ + /* ================================================================== */ + + /* Surfaces (backgrounds). */ + --bg: #ffffff; + --bg-page: #efefef; + --bg-sidebar: #343131; + --bg-topbar: #2a2828; + --bg-input: #ffffff; + --bg-key: #f0f0f0; + --kbd-bg: #f3f3f3; + --kbd-text: #6a6a6a; + --kbd-border: #c8c8c8; + --bg-search-highlight: #e1edf5; + --bg-hover: rgba(0, 0, 0, 0.04); + --bg-hover-on-dark: rgba(255, 255, 255, 0.08); + + /* Text. */ + --text: #404040; + --text-muted: #555555; + --text-secondary: #555555; + --text-on-dark: #d1d1d1; + --headings: #404040; + --ntuple-color: #333333; + + /* Sidebar: left nav is always on a dark surface; tokens here + are constant across light/dark mode (only --bg-sidebar shifts + between the two). */ + --sidebar-fg: #d9d9d9; + --sidebar-caption: #8aa8c0; + --sidebar-active-bg: #f0f0f0; + --sidebar-active-fg: #1a1a1a; + + /* Inline code (`literals`). */ + --inline-code-bg: rgba(175, 184, 193, 0.17); + --inline-code-text: inherit; + + /* Borders. */ + --border: #dddddd; + --scrollbar-thumb: #c1c1c1; + --border-radius: 6px; + --border-radius-code: 3px; + + --accent: #2371a8; + --accent-ring: rgba(40, 100, 180, 0.25); + + /* Inline API links (cross-refs to symbols). */ + --links-api: #336699; + --links-api-underline: #336699; + + /* Function/method signature box ("psutil.cpu_times(percpu=False)"). */ + --func-sig-bg: #f0f4f7; + --func-sig-border: #336699; + --func-sig-text: #404040; + --func-sig-name: #404040; + --func-sig-param: #555555; + + /* Lifted-surface accent (cards, sponsor logos). */ + --surface-raised: #f0f4f7; + + /* Admonitions (Note / Tip / Warning / Important / See also). */ + --adm-padding-y: 5px; + --adm-title: black; + --adm-note-bg: #f5f5f5; + --adm-note-border: #ccc; + --adm-note-accent-border: #2980b9; + --adm-tip-bg: #f0faf0; + --adm-tip-border: #8aba8a; + --adm-tip-icon: #4a8a4a; + --adm-warning-bg: #fff0f0; + --adm-warning-border: #e8a0a0; + --adm-warning-icon: #c0392b; + --adm-important-bg: #fdf3dc; + --adm-important-border: #d4a23e; + --adm-important-icon: #b07d17; + --adm-seealso-bg: #f3f9fe; + --adm-seealso-border: #b0cfe8; + --adm-seealso-accent-border: #336699; + --adm-seealso-link: #2a6099; + --adm-seealso-hover: #2980b9; + + /* Version directives (versionadded / versionchanged / deprecated). */ + --versionadded-border: #4fc464; + --versionadded-color: #296433; + --versionchanged-border: #f4e34c; + --versionchanged-color: #854826; + --deprecated-border: #f44c4e; + --deprecated-color: #9f3133; + + /* Tables. */ + --table-header-bg: #f0f4f7; + --table-header-text: var(--text); + --table-row-odd: #f8f9fa; + --table-row-even: #ffffff; + --table-border: var(--border); + + /* Changelog labels. */ + --label-critical-bg: #fdeaea; + --label-critical-text: #a3161a; + --label-critical-border: #f3c6c7; + --label-build-fail-bg: #fdf2ea; + --label-build-fail-text: #933e06; + --label-build-fail-border: #f4d3bd; + --label-memleak-bg: #fdf5d8; + --label-memleak-text: #7d5e00; + --label-memleak-border: #f2dc9c; + --label-breaking-bg: #f4eefe; + --label-breaking-text: #5c3fa3; + --label-breaking-border: #e0d4fa; + + /* ================================================================== */ + /* Fonts */ + /* ================================================================== */ + + --font-body: + "Inter", + system-ui, + -apple-system, + "Segoe UI", + Roboto, + "Helvetica Neue", + Arial, + "Noto Sans", + sans-serif; + --font-mono: + "JetBrains Mono", + ui-monospace, + SFMono-Regular, + Menlo, + Monaco, + Consolas, + "Liberation Mono", + "Courier New", + monospace; + --code-line-height: 1.5; + + /* ================================================================== */ + /* Aliases (semantic names used by component files) */ + /* ================================================================== */ + + --content-text: var(--text); + --code-font: var(--font-mono); + --links: var(--accent); + --surface-sunken: var(--bg-key); + --card-bg: var(--bg); + --card-border: var(--border); + --theme-transition: 0.2s ease; +} + +/* ====================================================================== */ +/* Dark mode */ +/* ====================================================================== */ + +/* + * Dark theme overrides only the color tokens. Layout / fonts are + * mode-agnostic. data-theme is set on by extrahead's inline + * script + theme-toggle.js. + */ + +[data-theme="dark"] { + color-scheme: dark; + + /* Surfaces. */ + --bg: #1a1a1a; + --bg-page: #0e0e0e; + --bg-sidebar: #212121; + --bg-topbar: #181818; + --bg-input: #2a2a2a; + --bg-key: #2a2a2a; + --kbd-bg: #3a3a3a; + --kbd-text: #d4d4d4; + --kbd-border: #5a5a5a; + --bg-search-highlight: rgba(225, 237, 245, 0.15); + --bg-hover: rgba(255, 255, 255, 0.06); + --sidebar-active-bg: #404040; + --sidebar-active-fg: #fff; + + /* Text. */ + --text: #c8c8c8; + --text-muted: #9b9b9b; + --text-secondary: #d1d1d1; + --headings: #e0e0e0; + --ntuple-color: #6db99a; + + /* Borders. */ + --border: #2e2e2e; + --scrollbar-thumb: #4a4a4a; + + /* Accent: lighter blue for sufficient contrast on dark. */ + --accent: #6ab0de; + --accent-ring: rgba(106, 176, 222, 0.35); + + /* API links + signature box. */ + --links-api: #7ecfff; + --links-api-underline: #7ecfff; + --func-sig-bg: #2c2c2c; + --func-sig-border: #7ecfff; + --func-sig-text: #e0e0e0; + --func-sig-name: #e0e0e0; + --func-sig-param: #d1d1d1; + --surface-raised: #2c2c2c; + + /* Inline code. */ + --inline-code-bg: rgba(255, 255, 255, 0.08); + --inline-code-text: #e6e6e6; + + /* Admonitions: dark mode overrides. */ + --adm-title: #ddd; + --adm-note-bg: #2a2a2a; + --adm-note-border: #4a4a4a; + --adm-note-accent-border: #6ab0de; + --adm-tip-bg: #1a2a1a; + --adm-tip-border: #3a6a3a; + --adm-tip-icon: #6aba6a; + --adm-warning-bg: #472424; + --adm-warning-border: #8a3030; + --adm-warning-icon: #e08a80; + --adm-important-bg: #2a200d; + --adm-important-border: #6e5328; + --adm-important-icon: #d4a23e; + --adm-seealso-bg: #1d2e40; + --adm-seealso-border: #2d5280; + --adm-seealso-accent-border: #5599cc; + --adm-seealso-link: #90c8ee; + --adm-seealso-hover: #8bbfe0; + + /* Version directives: dark mode overrides. */ + --versionadded-color: #5ec46e; + --versionchanged-color: #b0873a; + --deprecated-color: #d06060; + + /* Tables: dark mode overrides. */ + --table-header-bg: #2d2d2d; + --table-row-odd: #222222; + --table-row-even: #272727; + --table-border: #424242; + + /* Changelog labels. */ + --label-critical-bg: rgba(225, 29, 33, 0.16); + --label-critical-text: #ff8a8c; + --label-critical-border: rgba(225, 29, 33, 0.45); + --label-build-fail-bg: rgba(245, 104, 10, 0.14); + --label-build-fail-text: #f79f64; + --label-build-fail-border: rgba(245, 104, 10, 0.4); + --label-memleak-bg: rgba(251, 202, 4, 0.14); + --label-memleak-text: #e5b833; + --label-memleak-border: rgba(251, 202, 4, 0.4); + --label-breaking-bg: rgba(212, 197, 249, 0.14); + --label-breaking-text: #cbb9f7; + --label-breaking-border: rgba(212, 197, 249, 0.4); +} + +/* ====================================================================== */ +/* Theme transition */ +/* ====================================================================== */ + +body, +.page, +.article, +.right-sidebar, +.right-sidebar-title, +.topbar, +.footer, +.search-page-form, +ul.search > li, +.shortcut-flyout-panel { + transition: + background-color var(--theme-transition), + color var(--theme-transition), + border-color var(--theme-transition); +} + +.no-transition, +.no-transition *, +.no-transition *::before, +.no-transition *::after { + transition: none !important; +} + +/* Honor the OS "reduce motion" setting: drop transitions, animations + and smooth scrolling everywhere. */ +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + transition-duration: 0.01ms !important; + scroll-behavior: auto !important; + } +} + +/* ====================================================================== */ +/* Utilities */ +/* ====================================================================== */ + +/* .no-visited: suppresses the browser-default purple ":visited" color */ +.no-visited:visited, +.no-visited a:visited { + color: inherit; +} + +/* Thin, theme-aware scrollbars for every scrollable region (code + blocks, the TOC, tables, ...). */ +* { + scrollbar-width: thin; + scrollbar-color: var(--scrollbar-thumb) transparent; +} + +::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +::-webkit-scrollbar-thumb { + background: var(--scrollbar-thumb); + border-radius: 4px; +} + +::-webkit-scrollbar-track { + background: transparent; +} + +/* about.html page */ +.about-shot { + display: block; + max-width: 100%; + max-height: 560px; + width: auto; + height: auto; + margin: 0.4em 0 1.4em; + border: 1px solid var(--border); + border-radius: 6px; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.08); +} + +/* Two screenshots side by side (light vs dark); wrap on narrow columns. */ +.about-shot-pair { + display: flex; + flex-wrap: wrap; + gap: 12px; + margin: 0.4em 0 1.4em; +} + +.about-shot-pair .about-shot { + flex: 1 1 240px; + min-width: 0; + margin: 0; +} + +/* Flash when clicking on a link / anchor */ +@keyframes anchor-flash { + from { + background-color: var(--accent-ring); + } +} + +:target > h1, +:target > h2, +:target > h3, +:target > h4, +:target > h5, +:target > h6, +:target + h1, +:target + h2, +:target + h3, +:target + h4, +:target + h5, +:target + h6, +dt:target { + animation: anchor-flash 1.8s ease-out; +} + +/* Keyboard focus ring on interactive controls (both themes). The + browser default is nearly invisible on the dark topbar/sidebar. */ +.topbar-link:focus-visible, +.topbar-btn:focus-visible, +.topbar-hamburger:focus-visible, +.topbar-text:focus-visible, +.topbar-logo:focus-visible, +.left-sidebar a:focus-visible, +.right-sidebar a:focus-visible, +.home-feature-card:focus-visible, +.home-platform-pill:focus-visible, +.home-stat:focus-visible, +.prev-next a:focus-visible, +.blog-card-title a:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 2px; + border-radius: var(--border-radius); +} diff --git a/docs/_static/css/blog.css b/docs/_static/css/blog.css new file mode 100644 index 0000000000..537ce18d20 --- /dev/null +++ b/docs/_static/css/blog.css @@ -0,0 +1,414 @@ +/* + * psutil-sphinx-theme: blog sub-site. + * + * Loaded for every page; rules are scoped to blog pages via + * `body:has(.blog-section-banner)` (any page under the blog path + * renders the banner) or `.article:has(.post-meta-banner)` (only + * blog posts) so non-blog pages stay unaffected. + */ + +/* ---- Tokens ---------------------------------------------------------- */ + +:root { + --font-post-body: "Merriweather", Charter, Georgia, serif; + --blog-accent: #9a3412; /* warm rust, editorial */ + --blog-accent-deep: #7c2d12; +} + +[data-theme="dark"] { + --blog-accent: #fb923c; + --blog-accent-deep: #fdba74; +} + +/* ---- Section banner (top of every blog page) ------------------------ */ + +.blog-section-banner { + display: flex; + align-items: center; + gap: 0.7em; + flex-wrap: wrap; + padding: 0.3em 0; + margin: 1.5em 0 0.3em 0; + border-bottom: 1px solid var(--border); +} + +@media (min-width: 1280px) { + .main:has(.article-with-toc:not(.no-toc)) .blog-section-banner { + max-width: calc(100% - var(--right-sidebar-width) - 32px); + } +} + +.blog-section-banner .blog-rss { + margin-left: auto; +} + +.blog-section-label { + display: inline-flex; + align-items: center; + gap: 0.45em; +} + +.blog-section-icon { + grid-area: icon; + width: 1.7em; + height: 1.7em; + color: var(--blog-accent); + flex-shrink: 0; +} + +a.blog-section-title, +a.blog-section-title:visited { + grid-area: title; + font-size: 1.9em; + font-weight: 700; + letter-spacing: 0.04em; + color: var(--text); + text-decoration: none; + line-height: 1.1; +} + +a.blog-section-title:hover, +.blog-card-title a:hover { + color: var(--blog-accent); + text-decoration: none; +} + +.blog-section-subtitle { + grid-area: subtitle; + font-size: 0.85em; + color: var(--text-muted); + opacity: 0.8; + margin-top: 0.15em; +} + +/* RSS subscribe button (orange outlined). */ +a.blog-rss, +a.blog-rss:visited { + display: inline-flex; + align-items: center; + gap: 0.4em; + padding: 0.35em 0.75em; + background: transparent; + color: var(--blog-accent); + border: 1px solid var(--blog-accent); + border-radius: 4px; + text-decoration: none; + font-weight: 600; + font-size: 0.9em; + line-height: 1; + flex: 0 0 auto; +} + +a.blog-rss:hover { + background: color-mix(in srgb, var(--blog-accent) 12%, transparent); + color: var(--blog-accent); + text-decoration: none; +} + +.blog-rss-icon { + width: 1.1em; + height: 1.1em; + flex: 0 0 auto; + transform: translateY(-0.05em); +} + +/* ---- Blog listing (post cards) -------------------------------------- */ + +.blog-listing-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 1em; + margin-bottom: 1.5em; +} + +.blog-listing-title { + font-size: 2.2em; + margin: 0; +} + +.blog-listing-title-value, +.blog-listing-title-value:visited { + color: var(--text); + font-weight: 500; + border-bottom: none; + text-decoration: none; +} + +ol.blog-cards { + list-style: none; + margin: 0; + padding: 0; +} + +ol.blog-cards > li.blog-card { + list-style: none; + margin: 0; +} + +ol.blog-cards > li.blog-card + li.blog-card { + border-top: 1px solid var(--border); +} + +.blog-card { + display: grid; + grid-template-columns: 1fr; + grid-template-areas: + "date" + "title" + "summary" + "tags"; + padding: 1.6em 0; + margin: 0; + background: none; + border: none; +} + +.blog-card:first-child { + padding-top: 0; +} + +.blog-card-head { + display: contents; +} + +.blog-card-meta { + grid-area: date; + color: color-mix(in srgb, var(--text-muted) 75%, transparent); + font-size: 0.78em; + font-weight: 400; + text-transform: uppercase; + letter-spacing: 0.08em; + font-variant-numeric: tabular-nums; + margin: 0 0 0.05em 0; +} + +.blog-card-comments { + letter-spacing: normal; +} + +.blog-card-comments::before { + content: "\00b7"; + margin: 0 0.72em 0 0.6em; + letter-spacing: 0.08em; +} + +.blog-card-draft { + font-style: italic; +} + +.blog-card-titlerow { + grid-area: title; + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 0.75em; +} + +.blog-card .blog-card-title { + min-width: 0; + font-family: var(--font-body); + font-size: 1.3em; + font-weight: 700; + line-height: 1.3; + margin: 0 0 0.4em 0; + padding: 0; +} + +.blog-card-title a, +.blog-card-title a:visited { + text-decoration: none; + color: var(--text); +} + +/* Match the article banner's effective size (0.78em inside a 0.9em + banner = 0.702em) so the listing badge is identical to it. */ +.blog-card-titlerow .post-meta-featured { + font-size: 0.702em; +} + +.blog-card-summary { + grid-area: summary; + color: var(--text-muted); + font-family: var(--font-body); + font-size: 1.05em; + line-height: 1.6; + margin: 0.3em 0 0 0; +} + +.blog-card-summary a, +.blog-card-summary a:visited { + color: inherit; + text-decoration: none; + border-bottom: 1px dotted var(--border); +} + +.blog-card-summary a:hover { + color: var(--blog-accent); + border-bottom-color: var(--blog-accent); + text-decoration: none; +} + +.blog-card-summary code { + background: none; + border: none; + padding: 0; + color: inherit; + font-size: 0.92em; +} + +.blog-card-summary p:empty { + display: none; +} + +.blog-card-summary p { + margin: 0; +} + +.blog-card .blog-card-tags { + grid-area: tags; + justify-self: start; + list-style: none; + display: flex; + flex-wrap: wrap; + gap: 0.3em; + margin: 0.9em 0 0 0; + padding-left: 0; +} + +.blog-card-tags li { + list-style: none; + margin: 0; +} + +/* ---- Post meta banner (date + tags below the H1 of a post) --------- */ + +.post-meta-banner { + display: flex; + flex-wrap: wrap; + gap: 0.5em; + align-items: center; + margin: 0 0 2em 0; + padding: 0.4em 0 0 0; + font-size: 0.9em; + color: var(--text-muted); +} + +.post-meta-banner > *:not(:first-child)::before { + content: "·"; + color: var(--text-muted); + opacity: 0.6; + margin: 0 0.5em 0 0.15em; +} + +.post-meta-featured, +a.post-meta-featured, +a.post-meta-featured:visited { + padding: 0.15em 0.65em; + border-radius: 999px; + font-size: 0.78em; + font-weight: 600; + letter-spacing: 0.06em; + text-transform: uppercase; + line-height: 1.6; + color: var(--blog-accent); + background: color-mix(in srgb, var(--blog-accent) 12%, transparent); + border: 1px solid color-mix(in srgb, var(--blog-accent) 28%, transparent); + text-decoration: none; +} + +a.post-meta-featured:hover { + background: color-mix(in srgb, var(--blog-accent) 22%, transparent); + border-color: color-mix(in srgb, var(--blog-accent) 50%, transparent); +} + +.post-meta-author { + font-style: italic; +} + +.post-meta-date, +.post-meta-readtime { + font-variant-numeric: tabular-nums; +} + +.post-meta-tags { + display: inline-flex; + flex-wrap: wrap; + gap: 0.3em; + align-items: center; +} + +/* Shared tag-pill style: card listing + post banner. */ +.post-meta-tags a, +.post-meta-tags a:visited, +.blog-card-tags a, +.blog-card-tags a:visited { + display: inline-block; + padding: 0.05em 0.55em; + border-radius: 999px; + font-size: 0.82em; + line-height: 1.6; + color: var(--text-muted); + background: color-mix(in srgb, var(--text-muted) 7%, transparent); + text-decoration: none; +} + +.post-meta-tags a:hover, +.blog-card-tags a:hover { + color: var(--blog-accent); + background: color-mix(in srgb, var(--blog-accent) 10%, transparent); + text-decoration: none; +} + +/* ---- Blog post body: switch to serif typography -------------------- */ + +.article:has(.post-meta-banner) p, +.article:has(.post-meta-banner) li, +.article:has(.post-meta-banner) blockquote, +.article:has(.post-meta-banner) dd { + font-family: var(--font-post-body); + line-height: 1.75; +} + +.article:has(.post-meta-banner) h1 { + font-size: 2.1em; + line-height: 1.25; + margin-bottom: 0.4em; +} + +.article:has(.post-meta-banner) h2 { + font-size: 1.5em; + line-height: 1.3; + margin-top: 1.8em; + padding-top: 0.5em; +} + +.article:has(.post-meta-banner) h3 { + font-size: 1.2em; + line-height: 1.35; + margin-top: 1.5em; +} + +/* GitHub-style blockquotes inside posts. */ +.article:has(.post-meta-banner) blockquote { + margin: 1em 0; + padding: 0 1em; + border-left: 0.25em solid var(--border); + color: var(--text-muted); + background: transparent; + font-style: normal; +} + +.article:has(.post-meta-banner) blockquote p { + margin: 0.5em 0; +} + +/* ---- Blog-scoped accent: redefine the link tokens inside the blog + sub-site so every link picks up the rust hue without needing + per-element overrides. */ + +body:has(.blog-section-banner) { + --accent: var(--blog-accent); + --links-api: var(--blog-accent-deep); + --links-api-underline: var(--blog-accent-deep); +} diff --git a/docs/_static/css/changelog.css b/docs/_static/css/changelog.css new file mode 100644 index 0000000000..d874ee5134 --- /dev/null +++ b/docs/_static/css/changelog.css @@ -0,0 +1,52 @@ +/* + * psutil-sphinx-theme: changelog labels (:label:`critical` & co). + */ + +.cl-label { + display: inline-block; + padding: 0 7px; + border: 1px solid transparent; + border-radius: 999px; + font-family: var(--font-body); + font-size: 0.72em; + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; + line-height: 1.7; + white-space: nowrap; + vertical-align: 1px; +} + +.cl-label-critical { + background: var(--label-critical-bg); + color: var(--label-critical-text); + border-color: var(--label-critical-border); +} + +.cl-label-breaking { + background: var(--label-breaking-bg); + color: var(--label-breaking-text); + border-color: var(--label-breaking-border); +} + +.cl-label-build-fail { + background: var(--label-build-fail-bg); + color: var(--label-build-fail-text); + border-color: var(--label-build-fail-border); +} + +.cl-label-memleak { + background: var(--label-memleak-bg); + color: var(--label-memleak-text); + border-color: var(--label-memleak-border); +} + +/* Printers drop backgrounds, which would leave the white text of + .cl-label-critical invisible. */ +@media print { + .cl-label { + background: none !important; + color: #000 !important; + border-color: #666 !important; + } +} diff --git a/docs/_static/css/code.css b/docs/_static/css/code.css new file mode 100644 index 0000000000..28113a1b16 --- /dev/null +++ b/docs/_static/css/code.css @@ -0,0 +1,234 @@ +/* + Styles for code blocks: Pygments syntax highlighting (Monokai for + dark mode, custom palette for light). +*/ + +/* ================================================================== */ +/* Imported Monokai rules */ +/* ================================================================== */ + +[data-theme="dark"] .highlight { color: #F8F8F2; } +[data-theme="dark"] .highlight .hll { background-color: #49483e; } +[data-theme="dark"] .highlight .c { color: #959077; } /* Comment */ +[data-theme="dark"] .highlight .err { color: #ED007E; background-color: #1E0010; } /* Error */ +[data-theme="dark"] .highlight .esc { color: #F8F8F2; } /* Escape */ +[data-theme="dark"] .highlight .g { color: #F8F8F2; } /* Generic */ +[data-theme="dark"] .highlight .k { color: #66D9EF; } /* Keyword */ +[data-theme="dark"] .highlight .l { color: #AE81FF; } /* Literal */ +[data-theme="dark"] .highlight .n { color: #F8F8F2; } /* Name */ +[data-theme="dark"] .highlight .o { color: #F8F8F2; } /* Operator */ +[data-theme="dark"] .highlight .x { color: #F8F8F2; } /* Other */ +[data-theme="dark"] .highlight .p { color: #F8F8F2; } /* Punctuation */ +[data-theme="dark"] .highlight .ch { color: #959077; } /* Comment.Hashbang */ +[data-theme="dark"] .highlight .cm { color: #959077; } /* Comment.Multiline */ +[data-theme="dark"] .highlight .cp { color: #959077; } /* Comment.Preproc */ +[data-theme="dark"] .highlight .cpf { color: #959077; } /* Comment.PreprocFile */ +[data-theme="dark"] .highlight .c1 { color: #959077; } /* Comment.Single */ +[data-theme="dark"] .highlight .cs { color: #959077; } /* Comment.Special */ +[data-theme="dark"] .highlight .gd { color: #FF4689; } /* Generic.Deleted */ +[data-theme="dark"] .highlight .ge { color: #F8F8F2; font-style: italic; } /* Generic.Emph */ +[data-theme="dark"] .highlight .ges { color: #F8F8F2; font-weight: bold; font-style: italic; } /* Generic.EmphStrong */ +[data-theme="dark"] .highlight .gr { color: #F8F8F2; } /* Generic.Error */ +[data-theme="dark"] .highlight .gh { color: #F8F8F2; } /* Generic.Heading */ +[data-theme="dark"] .highlight .gi { color: #9ccfa5; } /* Generic.Inserted */ +/* .go, .gp: overridden in the REPL theme sections below */ +[data-theme="dark"] .highlight .gs { color: #F8F8F2; font-weight: bold; } /* Generic.Strong */ +[data-theme="dark"] .highlight .gu { color: #959077; } /* Generic.Subheading */ +[data-theme="dark"] .highlight .gt { color: #F8F8F2; } /* Generic.Traceback */ +[data-theme="dark"] .highlight .kc { color: #66D9EF; } /* Keyword.Constant */ +[data-theme="dark"] .highlight .kd { color: #66D9EF; } /* Keyword.Declaration */ +[data-theme="dark"] .highlight .kn { color: #FF4689; } /* Keyword.Namespace */ +[data-theme="dark"] .highlight .kp { color: #66D9EF; } /* Keyword.Pseudo */ +[data-theme="dark"] .highlight .kr { color: #66D9EF; } /* Keyword.Reserved */ +[data-theme="dark"] .highlight .kt { color: #66D9EF; } /* Keyword.Type */ +[data-theme="dark"] .highlight .ld { color: #d8c88a; } /* Literal.Date */ +[data-theme="dark"] .highlight .m { color: #b9a5e0; } /* Literal.Number */ +[data-theme="dark"] .highlight .s { color: #d8c88a; } /* Literal.String */ +[data-theme="dark"] .highlight .na { color: #9ccfa5; } /* Name.Attribute */ +[data-theme="dark"] .highlight .nb { color: #F8F8F2; } /* Name.Builtin */ +[data-theme="dark"] .highlight .nc { color: #9ccfa5; } /* Name.Class */ +[data-theme="dark"] .highlight .no { color: #66D9EF; } /* Name.Constant */ +[data-theme="dark"] .highlight .nd { color: #9ccfa5; } /* Name.Decorator */ +[data-theme="dark"] .highlight .ni { color: #F8F8F2; } /* Name.Entity */ +[data-theme="dark"] .highlight .ne { color: #9ccfa5; } /* Name.Exception */ +[data-theme="dark"] .highlight .nf { color: #9ccfa5; } /* Name.Function */ +[data-theme="dark"] .highlight .nl { color: #F8F8F2; } /* Name.Label */ +[data-theme="dark"] .highlight .nn { color: #F8F8F2; } /* Name.Namespace */ +[data-theme="dark"] .highlight .nx { color: #9ccfa5; } /* Name.Other */ +[data-theme="dark"] .highlight .py { color: #F8F8F2; } /* Name.Property */ +[data-theme="dark"] .highlight .nt { color: #FF4689; } /* Name.Tag */ +[data-theme="dark"] .highlight .nv { color: #F8F8F2; } /* Name.Variable */ +[data-theme="dark"] .highlight .ow { color: #66D9EF; } /* Operator.Word */ +[data-theme="dark"] .highlight .pm { color: #F8F8F2; } /* Punctuation.Marker */ +[data-theme="dark"] .highlight .w { color: #F8F8F2; } /* Text.Whitespace */ +[data-theme="dark"] .highlight .mb { color: #b9a5e0; } /* Literal.Number.Bin */ +[data-theme="dark"] .highlight .mf { color: #b9a5e0; } /* Literal.Number.Float */ +[data-theme="dark"] .highlight .mh { color: #b9a5e0; } /* Literal.Number.Hex */ +[data-theme="dark"] .highlight .mi { color: #b9a5e0; } /* Literal.Number.Integer */ +[data-theme="dark"] .highlight .mo { color: #b9a5e0; } /* Literal.Number.Oct */ +[data-theme="dark"] .highlight .sa { color: #d8c88a; } /* Literal.String.Affix */ +[data-theme="dark"] .highlight .sb { color: #d8c88a; } /* Literal.String.Backtick */ +[data-theme="dark"] .highlight .sc { color: #d8c88a; } /* Literal.String.Char */ +[data-theme="dark"] .highlight .dl { color: #d8c88a; } /* Literal.String.Delimiter */ +[data-theme="dark"] .highlight .sd { color: #d8c88a; } /* Literal.String.Doc */ +[data-theme="dark"] .highlight .s2 { color: #d8c88a; } /* Literal.String.Double */ +[data-theme="dark"] .highlight .se { color: #AE81FF; } /* Literal.String.Escape */ +[data-theme="dark"] .highlight .sh { color: #d8c88a; } /* Literal.String.Heredoc */ +[data-theme="dark"] .highlight .si { color: #d8c88a; } /* Literal.String.Interpol */ +[data-theme="dark"] .highlight .sx { color: #d8c88a; } /* Literal.String.Other */ +[data-theme="dark"] .highlight .sr { color: #d8c88a; } /* Literal.String.Regex */ +[data-theme="dark"] .highlight .s1 { color: #d8c88a; } /* Literal.String.Single */ +[data-theme="dark"] .highlight .ss { color: #d8c88a; } /* Literal.String.Symbol */ +[data-theme="dark"] .highlight .bp { color: #F8F8F2; } /* Name.Builtin.Pseudo */ +[data-theme="dark"] .highlight .fm { color: #9ccfa5; } /* Name.Function.Magic */ +[data-theme="dark"] .highlight .vc { color: #F8F8F2; } /* Name.Variable.Class */ +[data-theme="dark"] .highlight .vg { color: #F8F8F2; } /* Name.Variable.Global */ +[data-theme="dark"] .highlight .vi { color: #F8F8F2; } /* Name.Variable.Instance */ +[data-theme="dark"] .highlight .vm { color: #F8F8F2; } /* Name.Variable.Magic */ +[data-theme="dark"] .highlight .il { color: #b9a5e0; } /* Literal.Number.Integer.Long */ + +/* layout */ +div[class^="highlight-"] { + border-radius: var(--border-radius-code); + background: transparent; +} + +/* Padding: applied to the wrapper, none on the inner pre so we don't + double up margins or shrink the scrollbar area. */ +.highlight { + padding: 12px 14px; + overflow-x: auto; +} + +.highlight pre { + padding: 0; + margin: 0; +} + +/* font */ +.highlight, +.highlight pre { + font-size: 13px !important; + font-family: var(--code-font) !important; +} + +/* REPL >>> prompt */ +.highlight .gp { + font-weight: normal; + font-style: normal; +} + + /* REPL output */ +.highlight .go { + font-weight: normal; + font-style: normal; +} + +.highlight pre { + line-height: var(--code-line-height) !important; +} + +/* background */ +[data-theme="light"] .highlight { + background: #eef1f4 !important; + border: 1px solid #c7ccd1; + border-radius: var(--border-radius-code); +} + +[data-theme="dark"] .highlight { + background: #262626 !important; + border: 1px solid #3a3a3a; + border-radius: var(--border-radius-code); +} + +.highlight .k { font-weight:normal; font-style: normal !important;} /* keyword */ +.highlight .p { font-weight:normal; } /* (, [, ], )*/ +.highlight .mi { font-weight:normal; } /* numbers */ +.highlight .mf { font-weight:normal; } /* floats */ + +/* Avoiod != → ≠, <= → ≤, etc.. */ +.highlight, +.highlight pre, +code, +pre, +kbd, +samp, +tt { + font-variant-ligatures: none; +} + +/* ----------------- REPL light theme ---------------- */ + +[data-theme="light"] .highlight-pycon .pycon-number { color: #204A87; } +[data-theme="light"] .highlight-pycon .pycon-string { color: #8a5a2d; } +[data-theme="light"] .highlight .s, +[data-theme="light"] .highlight .s1, +[data-theme="light"] .highlight .s2, +[data-theme="light"] .highlight .sa, +[data-theme="light"] .highlight .sb, +[data-theme="light"] .highlight .sc, +[data-theme="light"] .highlight .dl, +[data-theme="light"] .highlight .sd, +[data-theme="light"] .highlight .sh, +[data-theme="light"] .highlight .si, +[data-theme="light"] .highlight .sx, +[data-theme="light"] .highlight .sr, +[data-theme="light"] .highlight .ss, +[data-theme="light"] .highlight .se { color: #4f6b39; } /* strings: muted olive */ +[data-theme="light"] .highlight-pycon .pycon-field { color: #4d4d4d; background: #eff1f3} +[data-theme="light"] .highlight .gp { color: #6a6a6a; } /* >>> prompt: muted */ +[data-theme="light"] .highlight .c, +[data-theme="light"] .highlight .ch, +[data-theme="light"] .highlight .cm, +[data-theme="light"] .highlight .cp, +[data-theme="light"] .highlight .cpf, +[data-theme="light"] .highlight .c1, +[data-theme="light"] .highlight .cs { color: #5b636d; font-style: normal; } /* comments: muted gray */ +[data-theme="light"] .highlight .go { color: #6a6a6a; } /* output dimmer than code */ + +/* ----------------- REPL dark theme ---------------- */ + +[data-theme="dark"] .highlight-pycon .pycon-number { color: #b9a5e0; } +[data-theme="dark"] .highlight-pycon .pycon-string { color: #d8c88a; } +[data-theme="dark"] .highlight-pycon .pycon-field { color: #9ccfa5; } +[data-theme="dark"] .highlight .gp { color: #888888; } /* >>> prompt */ +[data-theme="dark"] .highlight .go { color: #6c7a88; } /* output dimmer than code */ + +/* ----------------- syntax highlight ---------------- */ + +[data-theme="dark"] .highlight .k { color: #f56b8b} /* keyword */ +[data-theme="dark"] .highlight .kn { color: #f56b8b} /* keyword.namespace */ + +/* --------------------------------------------------- */ + +[data-theme="light"] .highlight .mi { color: #204A87} /* numbers */ +[data-theme="light"] .highlight .nf { color: #A31515} /* functions */ +[data-theme="light"] .highlight .o { color: inherit; font-weight: normal; } /* operators */ + +/* ================================================================== */ +/* sphinx-codeautolink (makes APIs in code blocks clickable) */ +/* ================================================================== */ + +a.sphinx-codeautolink-a, +a.sphinx-codeautolink-a:visited { + color: inherit; + text-decoration: none; + border-bottom: none; + border-radius: 3px; + transition: background var(--theme-transition), + color var(--theme-transition), + box-shadow var(--theme-transition); +} + +a.sphinx-codeautolink-a:hover { + color: var(--links); + background: color-mix(in srgb, var(--links) 18%, transparent); + box-shadow: 0 0 0 2px color-mix(in srgb, var(--links) 18%, transparent); + text-decoration: none; + border-bottom: none; +} + +[data-theme="dark"] a.sphinx-codeautolink-a:hover { + background: color-mix(in srgb, var(--links) 45%, transparent); + box-shadow: 0 0 0 2px color-mix(in srgb, var(--links) 45%, transparent); +} diff --git a/docs/_static/css/comments.css b/docs/_static/css/comments.css new file mode 100644 index 0000000000..ef8d0762be --- /dev/null +++ b/docs/_static/css/comments.css @@ -0,0 +1,23 @@ +/* Blog comments (giscus), below the prev/next nav. */ + +.comments { + margin-top: 2em; + padding-top: 1em; + border-top: 1px solid var(--border); +} + +.comments h2 { + margin-top: 0; + margin-bottom: 0.8em; + font-size: 1.3em; + color: var(--headings); + font-weight: 600; +} + +/* giscus sizes its own iframe; just keep it inside the column. */ +.comments .giscus, +.comments iframe.giscus-frame { + width: 100%; + max-width: 100%; + border: none; +} diff --git a/docs/_static/css/copy-page.css b/docs/_static/css/copy-page.css new file mode 100644 index 0000000000..9aebeacd09 --- /dev/null +++ b/docs/_static/css/copy-page.css @@ -0,0 +1,107 @@ +/* "Copy page" button appended to the h1 by js/copy-page.js. */ + +.article h1:has(button.copy-page) { + display: flex; + align-items: center; + flex-wrap: wrap; +} + +.article h1 button.copy-page { + margin-left: auto; + display: inline-flex; + align-items: center; + gap: 5px; + padding: 3px 8px; + border: 1px solid var(--border); + border-radius: 5px; + background: transparent; + color: var(--text-muted); + font-family: inherit; + font-size: 0.34em; + font-weight: 400; + line-height: 1.5; + letter-spacing: 0.01em; + white-space: nowrap; + cursor: pointer; + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.06); + transition: + background 0.15s, + color 0.15s, + border-color 0.15s, + box-shadow 0.15s; +} + +.article h1 button.copy-page:focus { + outline: none; +} + +.article h1 button.copy-page:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 2px; +} + +.article h1 button.copy-page:hover { + background: var(--bg-hover); + border-color: var(--text-muted); + color: var(--text); + box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1); +} + +.copy-page-icons { + display: inline-grid; + place-items: center; +} + +.copy-page-icon-copy, +.copy-page-icon-check { + grid-area: 1 / 1; +} + +.copy-page-icon-check { + visibility: hidden; + color: #2e9e4f; +} + +.copy-page.copied .copy-page-icon-copy { + visibility: hidden; +} + +.copy-page.copied .copy-page-icon-check { + visibility: visible; +} + +.copy-page.failed .copy-page-icon-copy { + color: #c0392b; +} + +.copy-page-labels { + display: inline-grid; + place-items: center start; +} + +.copy-page-label-copy, +.copy-page-label-done { + grid-area: 1 / 1; +} + +.copy-page-label-done { + visibility: hidden; +} + +.copy-page.copied .copy-page-label-copy { + visibility: hidden; +} + +.copy-page.copied .copy-page-label-done { + visibility: visible; +} + +@media (max-width: 600px) { + .article h1 button.copy-page .copy-page-labels { + display: none; + } + + .article h1 button.copy-page { + padding: 3px 6px; + } +} diff --git a/docs/_static/css/doc-icons.css b/docs/_static/css/doc-icons.css new file mode 100644 index 0000000000..57b2812a5b --- /dev/null +++ b/docs/_static/css/doc-icons.css @@ -0,0 +1,130 @@ +/* Left sidebar menu icons. Also shared with search results. */ + +/* Documentation */ +a[href*="install/"], +ul.search > li:has(> a[href*="install/"]), +body[data-page="install"] .left-sidebar a.current { + --doc-icon: "\f019"; +} + +a[href*="api-overview/"], +ul.search > li:has(> a[href*="api-overview/"]), +body[data-page="api-overview"] .left-sidebar a.current { + --doc-icon: "\f14e"; +} + +a[href*="api/"], +ul.search > li:has(> a[href*="api/"]), +body[data-page="api"] .left-sidebar a.current { + --doc-icon: "\f121"; +} + +a[href*="faq/"], +ul.search > li:has(> a[href*="faq/"]), +body[data-page="faq"] .left-sidebar a.current { + --doc-icon: "\3f"; +} + +a[href*="performance/"], +ul.search > li:has(> a[href*="performance/"]), +body[data-page="performance"] .left-sidebar a.current { + --doc-icon: "\f0e7"; +} + +a[href*="recipes/"], +ul.search > li:has(> a[href*="recipes/"]), +body[data-page="recipes"] .left-sidebar a.current { + --doc-icon: "\f2e7"; +} + +/* Reference */ +a[href*="shell-equivalents/"], +ul.search > li:has(> a[href*="shell-equivalents/"]), +body[data-page="shell-equivalents"] .left-sidebar a.current { + --doc-icon: "\f120"; +} + +a[href*="stdlib-equivalents/"], +ul.search > li:has(> a[href*="stdlib-equivalents/"]), +body[data-page="stdlib-equivalents"] .left-sidebar a.current { + --doc-icon: "\f3e2"; + --doc-icon-font: var(--fa-family-brands); + --doc-icon-weight: 400; +} + +a[href*="glossary/"], +ul.search > li:has(> a[href*="glossary/"]), +body[data-page="glossary"] .left-sidebar a.current { + --doc-icon: "\f031"; +} + +a[href*="platform/"], +ul.search > li:has(> a[href*="platform/"]), +body[data-page="platform"] .left-sidebar a.current { + --doc-icon: "\f5fd"; +} + +a[href*="migration/"], +ul.search > li:has(> a[href*="migration/"]), +body[data-page="migration"] .left-sidebar a.current { + --doc-icon: "\f362"; +} + +/* About */ +a[href*="adoption/"], +ul.search > li:has(> a[href*="adoption/"]), +body[data-page="adoption"] .left-sidebar a.current { + --doc-icon: "\f0c0"; +} + +a[href*="alternatives/"], +ul.search > li:has(> a[href*="alternatives/"]), +body[data-page="alternatives"] .left-sidebar a.current { + --doc-icon: "\e13a"; +} + +a[href*="funding/"], +ul.search > li:has(> a[href*="funding/"]), +body[data-page="funding"] .left-sidebar a.current { + --doc-icon: "\f4be"; +} + +a[href*="credits/"], +ul.search > li:has(> a[href*="credits/"]), +body[data-page="credits"] .left-sidebar a.current { + --doc-icon: "\f559"; +} + +/* Project */ +a[href*="blog/"], +ul.search > li:has(> a[href*="blog/"]), +ul.search > li:has(> a[href^="blog/"]), +ul.search > li:has(> a[href*="/blog/"]), +body[data-page="blog"] .left-sidebar a.current, +body[data-page^="blog/"] .left-sidebar li.toctree-l1 > a[href$="../"] { + --doc-icon: "\f09e"; +} + +a[href*="changelog/"], +ul.search > li:has(> a[href*="changelog/"]), +body[data-page="changelog"] .left-sidebar a.current { + --doc-icon: "\f1da"; +} + +a[href*="timeline/"], +ul.search > li:has(> a[href*="timeline/"]), +body[data-page="timeline"] .left-sidebar a.current { + --doc-icon: "\e29c"; +} + +a[href*="devguide/"], +ul.search > li:has(> a[href*="devguide/"]), +body[data-page="devguide"] .left-sidebar a.current { + --doc-icon: "\f7d9"; +} + +a[href*="genindex/"], +ul.search > li:has(> a[href*="genindex/"]), +body[data-page="genindex"] .left-sidebar a.current { + --doc-icon: "\f15d"; +} diff --git a/docs/_static/css/fontawesome.css b/docs/_static/css/fontawesome.css new file mode 100644 index 0000000000..48ca3d60fe --- /dev/null +++ b/docs/_static/css/fontawesome.css @@ -0,0 +1,101 @@ +/* + * FontAwesome 7.2.0, subset to the glyphs this theme actually uses + * (see _static/fonts/fa-*-subset.woff2). Don't depend on a third-party + * CDN. FontAwesome Free is CC BY 4.0 / SIL OFL 1.1. + */ + +@font-face { + font-family: "Font Awesome 7 Free"; + font-style: normal; + font-weight: 900; + font-display: block; + src: url(../fonts/fa-solid-subset.woff2) format("woff2"); +} + +@font-face { + font-family: "Font Awesome 7 Free"; + font-style: normal; + font-weight: 400; + font-display: block; + src: url(../fonts/fa-regular-subset.woff2) format("woff2"); +} + +@font-face { + font-family: "Font Awesome 7 Brands"; + font-style: normal; + font-weight: 400; + font-display: block; + src: url(../fonts/fa-brands-subset.woff2) format("woff2"); +} + +/* Family-name vars consumed by doc-icons.css / left-sidebar.css. */ +:root { + --fa-family-classic: "Font Awesome 7 Free"; + --fa-family-brands: "Font Awesome 7 Brands"; +} + +/* Base icon rendering (replaces FontAwesome's core rules). */ +.fa-solid, +.fa-regular, +.fa-brands { + display: inline-block; + font-style: normal; + font-variant: normal; + line-height: 1; + text-rendering: auto; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.fa-solid { + font-family: "Font Awesome 7 Free"; + font-weight: 900; +} + +.fa-regular { + font-family: "Font Awesome 7 Free"; + font-weight: 400; +} + +.fa-brands { + font-family: "Font Awesome 7 Brands"; + font-weight: 400; +} + +/* Glyphs referenced by class (the doc-icons set their codepoint via + --doc-icon and don't need entries here). */ +.fa-bars::before { + content: "\f0c9"; +} + +.fa-chevron-up::before { + content: "\f077"; +} + +.fa-tag::before { + content: "\f02b"; +} + +.fa-list-ul::before { + content: "\f0ca"; +} + +.fa-magnifying-glass::before { + content: "\f002"; +} + +.fa-sun::before { + content: "\f185"; +} + +.fa-moon::before { + content: "\f186"; +} + +.fa-github::before { + content: "\f09b"; +} + +.fa-python::before { + content: "\f3e2"; +} diff --git a/docs/_static/css/fonts.css b/docs/_static/css/fonts.css new file mode 100644 index 0000000000..3e3f49a921 --- /dev/null +++ b/docs/_static/css/fonts.css @@ -0,0 +1,87 @@ +/* + * Self-hosted web fonts (latin subset). Replaces the Google Fonts + * CDN so the docs render correctly without reaching Google (blocked + * in mainland China) and without leaking visitor IPs. OFL-1.1; see + * _static/fonts/LICENSE.txt. + */ + +@font-face { + font-family: 'Inter'; + font-style: normal; + font-weight: 400; + font-display: swap; + src: url(../fonts/inter-400.woff2) format('woff2'); + unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; +} + +@font-face { + font-family: 'Inter'; + font-style: normal; + font-weight: 500; + font-display: swap; + src: url(../fonts/inter-500.woff2) format('woff2'); + unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; +} + +@font-face { + font-family: 'Inter'; + font-style: normal; + font-weight: 600; + font-display: swap; + src: url(../fonts/inter-600.woff2) format('woff2'); + unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; +} + +@font-face { + font-family: 'Inter'; + font-style: normal; + font-weight: 700; + font-display: swap; + src: url(../fonts/inter-700.woff2) format('woff2'); + unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; +} + +@font-face { + font-family: 'JetBrains Mono'; + font-style: normal; + font-weight: 400; + font-display: swap; + src: url(../fonts/jetbrains-mono-400.woff2) format('woff2'); + unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; +} + +@font-face { + font-family: 'JetBrains Mono'; + font-style: normal; + font-weight: 600; + font-display: swap; + src: url(../fonts/jetbrains-mono-600.woff2) format('woff2'); + unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; +} + +@font-face { + font-family: 'Merriweather'; + font-style: italic; + font-weight: 400; + font-display: swap; + src: url(../fonts/merriweather-400-italic.woff2) format('woff2'); + unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; +} + +@font-face { + font-family: 'Merriweather'; + font-style: normal; + font-weight: 400; + font-display: swap; + src: url(../fonts/merriweather-400.woff2) format('woff2'); + unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; +} + +@font-face { + font-family: 'Merriweather'; + font-style: normal; + font-weight: 700; + font-display: swap; + src: url(../fonts/merriweather-700.woff2) format('woff2'); + unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; +} diff --git a/docs/_static/css/footer.css b/docs/_static/css/footer.css new file mode 100644 index 0000000000..42992bd092 --- /dev/null +++ b/docs/_static/css/footer.css @@ -0,0 +1,121 @@ +/* Footer sits in the content column (grid col 2) of .page, below the + main row. Padding matches .main so it lines up with the article. */ +.footer { + grid-column: 2; + margin-top: 0.5em; + padding: 1em 24px 16px; + border-top: 1px solid var(--border); +} + +/* Mobile: single-column grid, so the footer is in column 1. */ +@media (max-width: 1024px) { + .footer { + grid-column: 1; + padding: 1em 16px 16px; + } +} + +.footer-content { + display: flex; + align-items: center; + gap: 12px; +} + +.footer-text { + color: var(--text-muted); + line-height: 1.4; + display: flex; + flex-wrap: wrap; + gap: 0.3em 0.7em; +} + +/* Middot between items, attached to each item except the last so a + wrapped row never starts with an orphan separator. */ +.footer-text > span:not(:last-child)::after { + content: "·"; + margin-left: 0.7em; + color: var(--text-muted); + opacity: 0.7; +} + +.footer-text a, +.footer-text a:visited { + color: var(--text-muted); + text-decoration: underline; + text-decoration-color: var(--border); + text-underline-offset: 3px; +} + +.footer-text a:hover { + color: var(--accent); + text-decoration-color: currentColor; +} + +/* Easter egg: the © symbol links to the 404 page. No underline. */ +.footer-text a.footer-egg, +.footer-text a.footer-egg:visited { + color: inherit; + text-decoration: none; +} + +/* PyPI + RSS icon links, grouped at the right of the footer row. */ +.footer-icons { + margin-left: auto; + display: inline-flex; + align-items: center; + gap: 14px; + flex-shrink: 0; +} + +.footer-pypi, +.footer-pypi:visited, +.footer-github, +.footer-github:visited, +.footer-rss, +.footer-rss:visited { + display: inline-flex; + align-items: center; + color: var(--text-muted); + text-decoration: none; + opacity: 0.75; + transition: opacity 0.15s ease, color 0.15s ease; +} + +/* FontAwesome glyphs, sized to roughly match the RSS icon. The python + logo's ink is shorter than the others, so nudge it up to match. */ +.footer-github { + font-size: 1.3em; +} + +.footer-pypi { + font-size: 1.4em; +} + +.footer-rss { + font-size: 0.9em; +} + +.footer-pypi:hover, +.footer-github:hover, +.footer-rss:hover { + color: var(--accent); + opacity: 1; +} + +.footer-text a.footer-egg:hover { + color: var(--accent); +} + +/* RSS glyph from a single svg file, colored via mask so it follows + currentColor (muted, accent on hover). Shared with the blog banner. */ +.rss-icon { + display: inline-block; + background-color: currentColor; + -webkit-mask: url(../images/rss.svg) center / contain no-repeat; + mask: url(../images/rss.svg) center / contain no-repeat; +} + +.footer-rss .rss-icon { + width: 1.4em; + height: 1.4em; +} diff --git a/docs/_static/css/giscus.css b/docs/_static/css/giscus.css new file mode 100644 index 0000000000..7b0286fc0c --- /dev/null +++ b/docs/_static/css/giscus.css @@ -0,0 +1,131 @@ +/* + * Giscus widget theme. + */ + +@import url("https://giscus.app/themes/light.css") + screen and (prefers-color-scheme: light); +@import url("https://giscus.app/themes/dark.css") + screen and (prefers-color-scheme: dark); + +main { + --color-canvas-default: #ffffff; + --color-canvas-overlay: #ffffff; + --color-canvas-subtle: #f0f4f7; + --color-canvas-inset: #f0f4f7; + + --color-fg-default: #404040; + --color-fg-muted: #555555; + --color-fg-subtle: #555555; + + --color-border-default: #dddddd; + --color-border-muted: #dddddd; + + --color-accent-fg: #2371a8; + --color-accent-emphasis: #2371a8; + + --color-btn-text: #404040; + --color-btn-bg: #f0f4f7; + --color-btn-border: #dddddd; + --color-btn-hover-bg: #e5ebf0; + + /* The "Comment" button, green by default. */ + --color-btn-primary-bg: #2371a8; + --color-btn-primary-hover-bg: #1d5f8e; + --color-btn-primary-disabled-bg: #a9c8dd; + --color-btn-primary-border: #2371a8; + --color-btn-primary-text: #ffffff; +} + +@media (prefers-color-scheme: dark) { + main { + --color-canvas-default: #1a1a1a; + --color-canvas-overlay: #1a1a1a; + --color-canvas-subtle: #2c2c2c; + --color-canvas-inset: #2c2c2c; + + --color-fg-default: #c8c8c8; + --color-fg-muted: #9b9b9b; + --color-fg-subtle: #9b9b9b; + + --color-border-default: #2e2e2e; + --color-border-muted: #2e2e2e; + + --color-accent-fg: #6ab0de; + --color-accent-emphasis: #6ab0de; + + --color-btn-text: #c8c8c8; + --color-btn-bg: #2c2c2c; + --color-btn-border: #2e2e2e; + --color-btn-hover-bg: #383838; + + --color-btn-primary-hover-bg: #2a83c2; + --color-btn-primary-disabled-bg: #2c4a5e; + } +} + +main .gsc-reactions { + flex-direction: row; + align-items: center; + justify-content: flex-start; + gap: 0.5rem; +} + +main .gsc-reactions-count, +main .gsc-comments-count { + margin: 0; + font-size: 0.85rem; + font-weight: 400; + text-align: left; + color: var(--color-fg-muted); +} + +main .gsc-reactions > * { + flex: 0 0 auto !important; + justify-content: flex-start !important; + margin-top: 0 !important; +} + +main .gsc-main { + gap: 1rem; +} + +@media (min-width: 601px) { + main .gsc-main { + position: relative; + } + + main .gsc-reactions { + position: absolute; + top: 0; + right: 0; + width: auto; + margin: 0; + } + + main .gsc-main:has(.gsc-right-header) .gsc-reactions { + right: 9.5rem; + } +} + +main .gsc-comment-box-main { + margin-left: 0; + margin-right: 0; +} + +main .gsc-comment-box-textarea { + box-sizing: border-box; + width: 100%; + border-left: 0; + border-right: 0; + border-top: 0; + border-radius: 0; + border-bottom-style: solid; +} + +main .gsc-comment-box-textarea-extras { + display: none; +} + +main .gsc-comment-box-md-toolbar { + display: none; +} diff --git a/docs/_static/css/home.css b/docs/_static/css/home.css new file mode 100644 index 0000000000..5dd71e8e05 --- /dev/null +++ b/docs/_static/css/home.css @@ -0,0 +1,650 @@ +/* ================================================================== */ +/* Layout */ +/* ================================================================== */ + +.home-page .right-sidebar { + display: none; +} + +/* Hide next/prev buttons */ +.home-page .prev-next { + display: none; +} + +/* ================================================================== */ +/* Hero */ +/* ================================================================== */ + +.home-page .hero { + text-align: center; + padding: 1.5em 0 1em; +} + +.home-page .hero-title { + font-family: var(--font-body); + font-size: 3.5rem; + font-weight: 700; + color: var(--headings); + line-height: 1.1; + margin: 0 0 0.3em; + font-variant-ligatures: none; + letter-spacing: 0.02em; + display: flex; + align-items: flex-end; + justify-content: center; +} + +[data-theme="dark"] .home-page .hero-title { + color: #fff; +} + +.home-page .hero-logo { + height: 55px; + width: auto; + padding-right: 5px; +} + +.home-page .hero-title span { + position: relative; + bottom: -2px; +} + +.home-page .hero-subtitle { + font-family: var(--font-body); + font-size: 1.15rem; + color: var(--text-muted); + margin-bottom: 0.9em; +} + +.home-page .home-intro { + max-width: 720px; + margin: 0.4em auto 0; +} + +.home-page .home-intro p { + margin-bottom: 0; +} + +/* ================================================================== */ +/* Install one-liner */ +/* ================================================================== */ + +.home-page .home-install { + position: relative; + display: flex; + align-items: center; + gap: 0.6em; + width: max-content; + max-width: 100%; + margin: 2.5em auto 0; + padding: 0.45em 0.55em 0.45em 0.85em; + background: transparent; + border: 1px solid var(--border); + border-radius: 8px; + font-family: var(--code-font); + font-size: 0.95rem; + line-height: 1.4; + color: var(--content-text); + transition: border-color 0.2s ease; +} + +.home-page .home-install:hover { + border-color: var(--text-muted); +} + +.home-page .home-install.copied { + border-color: var(--links); +} + +.home-page .home-install-prompt { + color: var(--text-muted); + font-weight: 600; + user-select: none; +} + +.home-page .home-install-cmd { + background: transparent; + border: none; + padding: 0; + color: var(--content-text); + font-family: var(--code-font); + font-size: inherit; + white-space: nowrap; +} + +.home-page button.home-install-copy { + display: inline-flex; + align-items: center; + justify-content: center; + width: 1.9em; + height: 1.9em; + padding: 0; + border: none; + border-radius: 999px; + background: transparent; + color: var(--text-muted); + cursor: pointer; + transition: background 0.15s ease, color 0.15s ease; +} + +.home-page button.home-install-copy:hover { + background: var(--surface-sunken); + color: var(--content-text); +} + +/* Icon swap: show copy by default, check when .copied is set on the + wrapper. Both SVGs are stacked at the same spot via grid trick on + the button so the layout doesn't shift. */ +.home-page .home-install-icon-copy, +.home-page .home-install-icon-check { + grid-area: 1 / 1; +} + +.home-page button.home-install-copy { + display: inline-grid; + place-items: center; +} + +.home-page .home-install-icon-check { + visibility: hidden; + color: var(--links); +} + +.home-page .home-install.copied .home-install-icon-copy { + visibility: hidden; +} + +.home-page .home-install.copied .home-install-icon-check { + visibility: visible; +} + +/* "Copied" toast: floats above the button, fades + lifts in. */ +.home-page .home-install-toast { + position: absolute; + bottom: calc(100% + 6px); + right: 0.4em; + padding: 0.18em 0.55em; + background: var(--links); + color: #fff; + font-family: var(--font-body); + font-size: 0.72rem; + font-weight: 600; + letter-spacing: 0.04em; + border-radius: 4px; + opacity: 0; + transform: translateY(4px); + pointer-events: none; + transition: opacity 0.18s ease, transform 0.18s ease; +} + +.home-page .home-install.copied .home-install-toast { + opacity: 1; + transform: translateY(0); +} + +/* ================================================================== */ +/* Platform pills */ +/* ================================================================== */ + +.home-page hr:has(+ .home-platforms) { + margin-top: 32px; + margin-bottom: 16px; +} + +/* Soften the Sponsors H2 */ +.home-page h2 { + font-size: 1.4rem; + font-weight: 600; + color: var(--text-muted); + text-align: center; + border-bottom: none; + margin-top: 2.5em; + margin-bottom: 0.8em; + text-transform: uppercase; + letter-spacing: 0.12em; +} + +.home-page .home-platforms { + display: flex; + flex-direction: column; + align-items: center; + gap: 0.4em; + margin: 2.5em 0 0; +} + +.home-page a.home-platforms-label, +.home-page a.home-platforms-label:visited { + font-size: 0.72rem; + color: var(--text-muted); + font-weight: 700; + letter-spacing: 0.12em; + text-transform: uppercase; + text-decoration: none; + transition: color var(--theme-transition); +} + +.home-page a.home-platforms-label:hover { + color: var(--content-text); + text-decoration: none; +} + +.home-page .home-platforms-pills { + display: flex; + justify-content: center; + flex-wrap: wrap; + gap: 8px; +} + +.home-page a.home-platform-pill, +.home-page a.home-platform-pill:visited { + display: inline-block; + padding: 3px 12px; + border: 1px solid var(--border); + border-radius: 999px; + font-size: 0.82rem; + color: var(--text-muted); + background: var(--surface-sunken); + text-decoration: none; + transition: + border-color var(--theme-transition), + color var(--theme-transition), + background var(--theme-transition); +} + +.home-page a.home-platform-pill:hover { + border-color: var(--text-muted); + color: var(--content-text); + background: var(--surface-raised); + text-decoration: none; +} + +/* ================================================================== */ +/* Adoption stats banner */ +/* ================================================================== */ + +.home-page .home-stats { + display: flex; + justify-content: center; + flex-wrap: wrap; + gap: 56px; + margin: 1em 0em 1.5em; +} + +.home-page .home-section-label:has(+ .home-stats) { + margin-top: 4em; +} + +/* Quiet stats: just numbers + labels, no boxes. The figures speak; + the chrome was reading as Series-B-SaaS marketing strip. */ +.home-page a.home-stat, +.home-page a.home-stat:visited { + display: flex; + flex-direction: column; + align-items: center; + color: var(--content-text); + text-decoration: none; + transition: color var(--theme-transition); +} + +.home-page a.home-stat:hover .home-stat-num { + color: var(--links); +} + +.home-page .home-stat-num { + font-size: 1.4rem; + font-weight: 700; + color: var(--content-text); + line-height: 1.2; + transition: color var(--theme-transition); +} + +.home-page .home-stat-label { + font-size: 0.8rem; + color: var(--text-muted); + margin-top: 4px; + text-align: center; + text-transform: uppercase; + letter-spacing: 0.06em; +} + +/* ================================================================== */ +/* Feature cards */ +/* ================================================================== */ + +.home-page .home-feature-cards { + display: grid; + grid-template-columns: repeat(6, 1fr); + gap: 14px; + margin: 0; +} + +@media (max-width: 900px) { + .home-page .home-feature-cards { + grid-template-columns: repeat(3, 1fr); + } +} + +.home-page .home-feature-card { + position: relative; + border: 1px solid var(--card-border); + border-radius: 8px; + padding: 20px 14px; + text-align: center; + background: var(--card-bg); + text-decoration: none; + transition: box-shadow 0.15s, border-color 0.15s; + display: block; +} + +.home-page .home-feature-card:hover { + box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1); + border-color: var(--links); +} + +/* Hover-revealed arrow signals "this is a nav link, click to jump." */ +.home-page .home-feature-card::after { + position: absolute; + top: 8px; + right: 10px; + font-size: 0.9rem; + color: var(--text-muted); + opacity: 0; + transition: opacity 0.15s; +} + +.home-page .home-feature-card:hover::after { + opacity: 1; + color: var(--links); +} + +/* Small caps section labels (Runs on, Explore the API, Try it). + * Margins drive the page rhythm: 2.5em above each section, 0.9em + * caption-to-content. Sections themselves use margin: 0 so gaps + * don't stack. */ +.home-page .home-section-label { + font-size: 0.72rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.12em; + color: var(--text-muted); + text-align: center; + margin: 2.5em 0 0.9em; +} + +/* Section-label owns the caption-to-content gap; zero out the + * next sibling's top margin so they don't stack. */ +.home-page .home-section-label + .home-stats, +.home-page .home-section-label + .home-quickstart, +.home-page .home-section-label + .sponsor-table { + margin-top: 0; +} + +/* Extra breathing room above "Try it" so it anchors as the page's + * interactive moment rather than blending with neighboring labels. */ +.home-page .home-section-label:has(+ .home-quickstart) { + margin-top: 4em; +} + +.home-page .home-icon-svg { + width: 2.2rem; + height: 2.2rem; + display: block; + margin: 0 auto 10px; +} + +[data-theme="dark"] .home-page .home-icon-svg { + filter: brightness(1.45) saturate(0.7); +} + +.home-page .home-feature-title { + font-weight: 700; + font-size: 0.95rem; + margin-bottom: 10px; + color: var(--headings); +} + +@media (max-width: 480px) { + .home-page .home-feature-cards { + grid-template-columns: repeat(3, 1fr); + gap: 8px; + } + .home-page .home-feature-card { + padding: 12px 8px; + } + .home-page .home-icon-svg { + width: 1.6rem; + height: 1.6rem; + } + .home-page .home-feature-title { + font-size: 0.8rem; + } +} + +/* ================================================================== */ +/* Sponsors (shared with funding.rst) */ +/* ================================================================== */ + +.home-page #sponsors { + margin: 1.5rem -56px 20px; + padding: 1.5rem 56px 2rem; + background: #f8f9fa; + border-top: 1px solid var(--border); + text-align: center; +} + +/* Full-bleed band: on mobile .main's side padding shrinks, so match + the pull-out to it or the band overflows and the page drags + sideways. */ +@media (max-width: 1024px) { + .home-page #sponsors { + margin-left: -16px; + margin-right: -16px; + padding-left: 16px; + padding-right: 16px; + } +} + +@media (max-width: 600px) { + .home-page #sponsors { + margin-left: -12px; + margin-right: -12px; + padding-left: 12px; + padding-right: 12px; + } +} + +.home-page #sponsors > .home-section-label { + margin-top: 1.2em; +} + +[data-theme="dark"] .home-page #sponsors { + background: #1e1e1e; +} + +.home-page #sponsors h1 { + text-align: center; +} + +.sponsor-table { + margin-left: auto; + margin-right: auto; + max-width: 100%; +} + +.sponsor-table svg { + max-width: 100%; + height: auto; +} + +/* Give the logo row a subtle surface so it reads as an intentional + block rather than free-floating SVGs. */ +table.sponsor-table { + background: var(--surface-sunken); + border: 1px solid var(--border); + border-radius: var(--border-radius); + margin: 1em auto 0.5em auto; + padding: 0.8em 1.2em; + display: table; +} + +table.sponsor-table, +table.sponsor-table tr, +table.sponsor-table td { + background: transparent; + border: none; +} + +.sponsor-cta-wrap { + text-align: center; + margin-top: 1em; +} + +a.sponsor-cta, +a.sponsor-cta:visited { + font-size: 0.85rem; + color: var(--text-muted); + text-decoration: underline dashed; + text-decoration-color: var(--border); + text-underline-offset: 3px; + transition: + color var(--theme-transition), + opacity var(--theme-transition), + text-decoration-color var(--theme-transition); +} + +a.sponsor-cta:hover { + color: var(--links); + opacity: 1; + text-decoration: underline; + text-decoration-color: var(--links); +} + +/* ================================================================== */ +/* Misc */ +/* ================================================================== */ + +/* TOC (hidden) */ +.home-page .toctree-wrapper { + display: none; +} + +/* ================================================================== */ +/* Quickstart tabs (sphinx-design) */ +/* ================================================================== */ + +.home-page .home-quickstart .sd-tab-set { + margin: 0; +} + +.home-page .home-quickstart .sd-tab-content { + padding: 0.5rem 0 0; + box-shadow: 0 -0.0625rem var(--border); +} + +/* "See more →" link below each tab's code block. */ +.home-quickstart .home-tab-more { + margin-top: 0.8em; + text-align: right; + font-size: 0.9rem; + font-weight: 600; +} + +.home-quickstart .home-tab-more p { + margin-bottom: 0; +} + +.home-quickstart .home-tab-more a, +.home-quickstart .home-tab-more a:visited { + color: var(--text-muted); + text-decoration: none; + transition: color 0.15s; +} + +.home-quickstart .home-tab-more a:hover { + color: var(--links); + text-decoration: underline; +} + +/* Inactive tabs: lighter than default so the current tab pops. */ +.home-quickstart .sd-tab-label { + color: var(--text-muted); + font-weight: 500; + transition: opacity 0.15s, color 0.15s; +} + +.home-quickstart .sd-tab-label:hover { + opacity: 1; + color: var(--content-text); +} + +/* Active tab: bolder weight + full opacity so it carries the page. */ +.home-quickstart input:checked + .sd-tab-label { + color: var(--links); + opacity: 1; + font-weight: 700; +} + +/* ================================================================== */ +/* Quickstart code-block window chrome */ +/* ================================================================== */ + +/* macOS-style traffic-light dots. */ + +.home-quickstart .highlight { + position: relative; + padding: 42px 18px 10px; +} + +.home-quickstart .highlight pre { + margin-top: 0; + padding-top: 0; +} + +.home-quickstart .highlight, +.home-quickstart .highlight pre { + font-size: 14px !important; + line-height: 1.65 !important; +} + +.home-quickstart .highlight button.copybtn { + top: 36px; + right: 8px; +} + +.home-quickstart .highlight::before { + content: ""; + position: absolute; + top: 0; + left: 0; + right: 0; + height: 28px; + border-top-left-radius: var(--border-radius); + border-top-right-radius: var(--border-radius); + background-color: #dde2e8; + background-image: + radial-gradient(circle at 14px 14px, #d49995 4px, transparent 4.5px), + radial-gradient(circle at 30px 14px, #d4b685 4px, transparent 4.5px), + radial-gradient(circle at 46px 14px, #9bc7a0 4px, transparent 4.5px); + border-bottom: 1px solid var(--border); + pointer-events: none; +} + +[data-theme="dark"] .home-quickstart .highlight::before { + background-color: #1d1d1d; + border-bottom-color: #3a3a3a; +} + +.home-quickstart .highlight::after { + content: "python"; + position: absolute; + top: 0; + right: 12px; + height: 28px; + line-height: 28px; + font-family: var(--font-mono); + font-size: 0.72rem; + letter-spacing: 0.04em; + color: var(--text-muted); + opacity: 0.7; + pointer-events: none; +} diff --git a/docs/_static/css/layout.css b/docs/_static/css/layout.css new file mode 100644 index 0000000000..e6b86f1e8c --- /dev/null +++ b/docs/_static/css/layout.css @@ -0,0 +1,294 @@ +body { + margin: 0; +} + +html { + /* Extra room below the topbar height so anchored headings clear + the topbar's drop shadow instead of landing inside it. */ + scroll-padding-top: calc(var(--header-height) + 0.5rem); +} + +/* ---- Header (banner + topbar), fixed at the top of the viewport ----- */ + +/* Height varies with the banner, so js/dev-banner.js keeps + --header-height in sync; everything that sits below the header + offsets by it. */ +.header-stack { + position: fixed; + top: 0; + left: 0; + right: 0; + z-index: 100; +} + +.topbar { + height: var(--topbar-height); +} + +/* ---- Page grid (sidebar + main column), centered -------------------- */ + +.page { + display: grid; + grid-template-columns: + var(--left-sidebar-width) + minmax(0, var(--content-max-width)); + max-width: var(--layout-width); + margin: 0 auto; + padding-top: var(--header-height); + background: var(--bg); +} + +.main { + min-width: 0; + padding: 0 24px; +} + +/* ---- Article + right sidebar grid (inside .main) -------------------- */ + +.article-with-toc { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 32px; + padding: 24px 0 24px 16px; + min-height: calc(100vh - var(--header-height)); +} + +/* No-TOC pages don't get the extra left indent; index.html etc. + handle their own horizontal padding via .article-column below. */ +.article-with-toc.no-toc { + padding-left: 0; +} + +@media (min-width: 1280px) { + .article-with-toc { + grid-template-columns: + minmax(0, 1fr) + var(--right-sidebar-width); + } + + .article-with-toc.no-toc { + grid-template-columns: minmax(0, 1fr); + } + + /* No-TOC pages get extra horizontal padding so prose isn't + hugging the wide column edge. */ + .article-with-toc.no-toc > .article-column { + padding-left: 32px; + padding-right: 32px; + } +} + +.article-column { + min-width: 0; +} + +.article { + min-width: 0; +} + +/* ---- Mobile: collapse to single column, sidebar slides in --------- */ + +@media (max-width: 1024px) { + .page { + grid-template-columns: minmax(0, 1fr); + } + + .sidebar-backdrop { + position: fixed; + inset: var(--header-height) 0 0 0; + background: rgba(0, 0, 0, 0.4); + z-index: 80; + opacity: 0; + pointer-events: none; + transition: opacity 0.2s ease; + } + + body.sidebar-open .sidebar-backdrop { + opacity: 1; + pointer-events: auto; + } + + /* Don't let the page scroll behind an open sidebar. */ + body.sidebar-open { + overflow: hidden; + } + + .main { + padding: 0 16px; + } + + .article-with-toc { + padding: 16px 0; + } +} + +@media (max-width: 600px) { + :root { + --topbar-height: 48px; + } + + .main { + padding: 0 12px; + } +} + +/* ---- Keyboard shortcut flyout (triggered by "?") ----------------- */ + +.shortcut-flyout { + position: fixed; + inset: 0; + z-index: 200; + display: none; +} + +.shortcut-flyout.is-open { + display: block; +} + +.shortcut-flyout-backdrop { + position: absolute; + inset: 0; + background: rgba(0, 0, 0, 0.45); +} + +.shortcut-flyout-panel { + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + box-sizing: border-box; + min-width: min(320px, 90vw); + max-width: 90vw; + background: var(--bg); + color: var(--text); + border: 1px solid var(--border); + border-radius: 8px; + box-shadow: 0 12px 36px rgba(0, 0, 0, 0.35); + padding: 1.2em 1.4em 1.4em; +} + +.shortcut-flyout-title { + font-size: 0.78rem; + font-weight: 700; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-muted); + margin-bottom: 0.8em; +} + +.shortcut-flyout-list { + display: grid; + grid-template-columns: auto 1fr; + gap: 0.55em 1.2em; + margin: 0; +} + +.shortcut-flyout-list dt { + margin: 0; + padding: 0; + background: transparent; + border: 0; + display: inline-flex; + align-items: center; + gap: 4px; + white-space: nowrap; + font-weight: 400; +} + +.shortcut-flyout-list dd { + margin: 0; + padding: 0; + color: var(--text); + align-self: center; +} + +/* Pill style is shared with the search-input hint + (.search-kbd kbd in search.css). */ + +/* ---- "Back to top" floating button ------------------------------- */ + +.back-to-top { + position: fixed; + right: max(20px, calc((100vw - var(--layout-width)) / 2 + 20px)); + bottom: 24px; + z-index: 90; + display: inline-flex; + align-items: center; + justify-content: center; + width: 34px; + height: 34px; + padding: 0; + border: 0; + border-radius: 999px; + background: var(--bg-topbar); + color: var(--text-on-dark); + font-size: 13px; + cursor: pointer; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12); + /* Visible state below uses opacity 0.55, muted at rest so it + doesn't grab attention; pops on hover. */ + opacity: 0; + pointer-events: none; + transition: + opacity 0.2s ease, + background 0.15s ease, + box-shadow 0.15s ease; +} + +.back-to-top.is-visible { + opacity: 0.55; + pointer-events: auto; +} + +/* Dark mode: --bg-topbar (#181818) blends into the page bg. Use a + lighter gray + full opacity so the button stands out. */ +[data-theme="dark"] .back-to-top { + background: #525252; + color: #f5f5f5; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.5); +} + +[data-theme="dark"] .back-to-top.is-visible { + opacity: 1; +} + +/* Mobile: bump size + opacity. The 28px gap-tucked button is way + too small/subtle for touch + small screens. */ +@media (max-width: 1024px) { + .back-to-top { + right: 14px; + bottom: 14px; + width: 36px; + height: 36px; + font-size: 13px; + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.22); + } + + .back-to-top.is-visible, + [data-theme="dark"] .back-to-top.is-visible { + opacity: 1; + } +} + +/* Wide viewports: tuck the button into the 32px gap between the + article column and the right TOC. Button is 28px, so 2px clear + on each side. */ +@media (min-width: 1280px) { + .back-to-top { + right: calc( + max(0px, (100vw - var(--layout-width)) / 2) + + var(--right-sidebar-width) + + 44px + ); + } +} + +.back-to-top.is-visible:hover { + opacity: 1; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.2); +} + +.back-to-top:focus-visible { + outline: 2px solid var(--accent-ring); + outline-offset: 3px; +} diff --git a/docs/_static/css/left-sidebar.css b/docs/_static/css/left-sidebar.css new file mode 100644 index 0000000000..ef571ae514 --- /dev/null +++ b/docs/_static/css/left-sidebar.css @@ -0,0 +1,115 @@ +.left-sidebar { + position: sticky; + top: var(--header-height); + align-self: start; + height: calc(100vh - var(--header-height)); + overflow-y: auto; + background: var(--bg-sidebar); + color: var(--sidebar-fg); +} + +/* ---- Captions ("Documentation", "Reference", ...) ------------------- */ + +.left-sidebar p.caption { + margin: 16px 0 4px; + padding: 0 1.618em; + font-size: 12px; + font-weight: 500; + letter-spacing: 0.18em; + text-transform: uppercase; + color: var(--sidebar-caption); +} + +.left-sidebar p.caption:first-of-type { + margin-top: 0; +} + +/* ---- List structure ------------------------------------------------- */ + +.left-sidebar ul, +.left-sidebar li { + list-style: none; + margin: 0; + padding: 0; +} + +/* ---- Items ---------------------------------------------------------- */ + +.left-sidebar a, +.left-sidebar a:visited { + display: block; + padding: 0.32em 1.2em; + line-height: 1.4; + color: var(--sidebar-fg); + text-decoration: none; +} + +.left-sidebar li.toctree-l1 > a { + text-transform: capitalize; +} + +.left-sidebar a:hover { + background: rgba(255, 255, 255, 0.04); +} + +/* ---- Active item ---------------------------------------------------- */ + +.left-sidebar li.current > a, +.left-sidebar li.current > a:visited { + background: var(--sidebar-active-bg); + color: var(--sidebar-active-fg); + box-shadow: + inset 3px 0 0 var(--bg-sidebar), + inset -3px 0 0 var(--bg-sidebar); +} + +/* ---- Mobile: off-canvas drawer ------------------------------------- */ + +@media (max-width: 1024px) { + .left-sidebar { + position: fixed; + top: var(--header-height); + left: 0; + z-index: 90; + width: var(--left-sidebar-width); + max-width: 85vw; + height: calc(100vh - var(--header-height)); + transform: translateX(-100%); + transition: transform 0.2s ease; + } + + body.sidebar-open .left-sidebar { + transform: translateX(0); + box-shadow: 4px 0 16px rgba(0, 0, 0, 0.35); + } +} + +/* Touch devices: taller nav rows for comfortable tapping. 36px keeps + them well above the WCAG 2.5.8 (AA) 24px floor without the airy gaps + the AAA 44px target leaves around single-line items. */ +@media (pointer: coarse) { + .left-sidebar a { + display: flex; + align-items: center; + min-height: 36px; + } +} + +/* ---- Per-item icons (Font Awesome) ---------------------------------- */ + +.left-sidebar li.toctree-l1 > a::before { + font-family: var(--doc-icon-font, var(--fa-family-classic)); + font-weight: var(--doc-icon-weight, 900); + display: inline-block; + width: 1.3em; + margin-right: 0.7em; + font-size: 0.85em; + text-align: center; + color: var(--sidebar-caption); + opacity: 0.85; + content: var(--doc-icon, ""); +} + +.left-sidebar li.current > a::before { + color: var(--sidebar-active-fg); +} diff --git a/docs/_static/css/prev-next.css b/docs/_static/css/prev-next.css new file mode 100644 index 0000000000..62e509d863 --- /dev/null +++ b/docs/_static/css/prev-next.css @@ -0,0 +1,86 @@ +/* + * psutil-sphinx-theme: prev / next page navigation at the bottom. + */ + +.prev-next { + display: flex; + justify-content: space-between; + align-items: flex-start; + gap: 16px; + margin-top: 1.5em; + padding: 12px 0 4px; + border-top: 1px solid var(--border); +} + +.prev-next-prev, +.prev-next-next { + display: inline-flex; + align-items: center; + gap: 12px; + flex: 1 1 0; + min-width: 0; + color: inherit; + text-decoration: none; + padding: 4px 8px; + border-radius: 4px; + transition: background 0.15s; +} + +.prev-next-next { + justify-content: flex-end; +} + +.prev-next-prev:hover, +.prev-next-next:hover { + background: var(--bg-hover); + text-decoration: none; +} + +/* Push next-only (no prev) to the right edge. */ +.prev-next-next:only-child { + margin-left: auto; +} + +.prev-next-block { + display: inline-flex; + flex-direction: column; + justify-content: center; + gap: 8px; + line-height: 1.2; + min-width: 0; +} + +.prev-next-prev .prev-next-block { + align-items: flex-start; + text-align: left; +} + +.prev-next-next .prev-next-block { + align-items: flex-end; + text-align: right; +} + +.prev-next-label { + font-size: 0.7rem; + font-weight: 500; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--text-muted); + opacity: 0.85; +} + +.prev-next-title { + font-size: 1rem; + font-weight: 400; + color: var(--headings); + overflow-wrap: anywhere; +} + +.prev-next-arrow { + display: inline-flex; + align-items: center; + align-self: stretch; + color: var(--text-muted); + font-size: 1.6rem; + line-height: 1; +} diff --git a/docs/_static/css/print.css b/docs/_static/css/print.css new file mode 100644 index 0000000000..fde7015697 --- /dev/null +++ b/docs/_static/css/print.css @@ -0,0 +1,54 @@ +/* Print / PDF: strip top bar and other useless parts. */ + +@media print { + .topbar, + .left-sidebar, + .right-sidebar, + .sidebar-backdrop, + .back-to-top, + .prev-next, + .shortcut-flyout, + .comments { + display: none !important; + } + + .page { + display: block; + max-width: none; + margin: 0; + padding-top: 0; + } + + .main { + padding: 0; + } + + .article-with-toc { + display: block; + padding: 0; + min-height: 0; + } + + body, + .article { + color: #000; + background: #fff; + } + + .article a[href^="http"]::after { + content: " (" attr(href) ")"; + font-size: 0.85em; + word-break: break-all; + } + + .highlight { + border: 1px solid #ccc !important; + background: #fff !important; + } + + pre, + .highlight { + white-space: pre-wrap; + overflow: visible; + } +} diff --git a/docs/_static/css/right-sidebar.css b/docs/_static/css/right-sidebar.css new file mode 100644 index 0000000000..2e257e8c17 --- /dev/null +++ b/docs/_static/css/right-sidebar.css @@ -0,0 +1,225 @@ +/* + * psutil-sphinx-theme: right sidebar (per-page TOC). + * + * Hidden below 1280px. Sticky inside its grid cell at wider widths. + * Sizes / spacing kept faithful to the previous psutil site. + */ + +.right-sidebar { + display: none; +} + +@media (max-width: 1279px) { + .topbar-toc-toggle { + display: none; + } + + body:has(.right-sidebar) .topbar-toc-toggle { + display: inline-flex; + } + + .right-sidebar { + display: block; + position: fixed; + top: var(--header-height); + right: 0; + z-index: 90; + width: var(--left-sidebar-width); + max-width: 85vw; + height: calc(100vh - var(--header-height)); + overflow-y: auto; + box-sizing: border-box; + padding: 16px; + background: var(--bg); + border-left: 1px solid var(--border); + transform: translateX(100%); + transition: transform 0.2s ease; + } + + body.toc-open .right-sidebar { + transform: translateX(0); + box-shadow: -4px 0 16px rgba(0, 0, 0, 0.35); + } +} + +@media (min-width: 1280px) { + .right-sidebar { + display: block; + position: sticky; + top: calc(var(--header-height) + 20px); + align-self: start; + max-height: calc(100vh - var(--header-height) - 36px); + overflow-y: auto; + padding: 0 16px; + margin-right: -24px; + border-left: 1px solid #ccc; + box-sizing: border-box; + font-size: 0.85rem; + line-height: 1.4; + } + + [data-theme="dark"] .right-sidebar { + border-left-color: #3a3a3a; + } +} + +/* ---- Title ("On this page") ---------------------------------------- */ + +.right-sidebar-title { + position: sticky; + top: 0; + z-index: 1; + background: var(--bg); + color: var(--text-muted); + font-size: 11px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.1em; + padding: 0 0 0.6rem 8px; +} + +/* {{ toc }} renders the H1 as the root
  • wrapping all sections. + Sphinx has no config to skip it; alternatives are a custom doctree + walker (Furo's approach) or this 2-rule CSS hide. */ +.right-sidebar > ul > li:first-child > a { + display: none; +} +.right-sidebar > ul > li:first-child > ul { + padding-left: 0; +} + +/* ---- List structure ------------------------------------------------ */ + +.right-sidebar ul, +.right-sidebar li { + list-style: none; + margin: 0; + padding: 0; +} + +.right-sidebar li { + margin: 4px 0; + font-size: 0.85rem; + line-height: 1.4; +} + +/* Nested uls don't add indent; indent is applied per-anchor instead, + so the active bar stays at column-edge regardless of depth. */ +.right-sidebar ul ul { + padding-left: 0; +} + +/* ---- Anchors ------------------------------------------------------- */ + +.right-sidebar a, +.right-sidebar a:visited { + display: block; + font-variant-numeric: tabular-nums; + padding: 0.2rem 8px; + margin: 0 6px 0 0; + color: var(--text-muted); + text-decoration: none; + overflow-wrap: anywhere; +} + +.right-sidebar a:hover { + color: var(--accent); + text-decoration: none; +} + +.right-sidebar a code { + font-weight: inherit; + background: none; + padding: 0; + color: inherit; +} + +/* H2 entries are emphasized; H3 styled as quiet group captions; H4 as items. */ +.right-sidebar > ul > li:first-child > ul > li > a { + font-size: 0.92rem; + font-weight: 600; +} + +/* Flat TOC: if only H2s, no H3+ anywhere) drop the bold. */ +.right-sidebar > ul > li:first-child > ul:not(:has(li > ul)) > li > a { + font-size: 0.85rem; + font-weight: 400; +} + +.right-sidebar > ul > li:first-child > ul > li + li { + margin-top: 0.5rem; +} + +.right-sidebar > ul > li:first-child > ul > li > ul > li:has(> ul) > a, +.right-sidebar + > ul > li:first-child + > ul > li > ul > li:has(> ul) + > a:visited { + font-weight: 600; + color: var(--text); +} + +/* Suppress the accent-blue hover flash on group rows (labels, not + primary navigation targets). */ +.right-sidebar > ul > li:first-child > ul > li > ul > li:has(> ul) > a:hover { + color: var(--text); +} + +.right-sidebar > ul > li:first-child > ul > li > ul > li:has(> ul) { + margin-top: 0.5rem; +} + +.right-sidebar > ul > li:first-child > ul > li > ul > li > ul > li > a { + font-size: 0.85rem; + font-weight: 400; +} + +.right-sidebar + > ul > li:first-child + > ul > li > ul > li > ul > li > ul > li > a { + font-size: 0.8rem; + font-weight: 400; +} + +/* Glossary TOC: populated by _ext/glossary_toc.py. */ +.right-sidebar ul.glossary-toc li { + margin: 2px 0; +} +.right-sidebar ul.glossary-toc a { + font-size: 0.85rem; + padding: 0.15rem 0 0.15rem 8px; +} + +.right-sidebar ul.blog-tags li { + margin: 2px 0; +} + +.right-sidebar ul.blog-tags a { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.6em; + font-size: 0.9rem; + padding: 0.15rem 0 0.15rem 8px; +} + +.right-sidebar .blog-tag-count { + flex: none; + min-width: 1.6em; + padding: 0.05em 0.5em; + border-radius: 999px; + background: var(--surface-raised); + color: var(--text-muted); + font-size: 0.75rem; + font-variant-numeric: tabular-nums; + text-align: center; +} + +/* ---- Active state (driven by js/right-toc.js) ---------------------- */ + +.right-sidebar li.scroll-current-leaf > a, +.right-sidebar li.scroll-current-leaf > a:visited { + color: var(--text); + background: var(--surface-raised); + border-radius: var(--border-radius); +} diff --git a/docs/_static/css/search.css b/docs/_static/css/search.css new file mode 100644 index 0000000000..bfae6620eb --- /dev/null +++ b/docs/_static/css/search.css @@ -0,0 +1,309 @@ +/* ====================================================================== */ +/* Sidebar search box */ +/* ====================================================================== */ + +.search-box { + position: sticky; + top: 0; + z-index: 5; + background: inherit; + padding: 12px; +} + +/* Dim the rest of the page while the search input is focused. */ +.search-overlay { + position: fixed; + inset: var(--header-height) 0 0 0; + background: rgba(0, 0, 0, 0.5); + opacity: 0; + pointer-events: none; + z-index: 95; + transition: opacity 0.15s ease; +} + +body.search-focused .search-overlay { + opacity: 1; +} + +body.search-focused .left-sidebar { + z-index: 96; +} + +body.search-focused .left-sidebar > :not(.search-box) { + opacity: 0.35; + transition: opacity 0.15s ease; +} + +.search-box form { + position: relative; + display: flex; + align-items: center; +} + +.search-box input[type="text"] { + flex: 1; + width: 100%; + box-sizing: border-box; + padding: 4px 60px 4px 12px; + font: inherit; + font-size: 13px; + border: 1px solid #555; + border-radius: 8px; + background: var(--bg-input); + color: var(--text); + transition: border-color 0.2s ease, box-shadow 0.2s ease; +} + +.search-box input[type="text"]:focus { + outline: none; + border-color: #7ab3d4; + box-shadow: 0 0 0 2px rgba(122, 179, 212, 0.3); +} + +.search-kbd { + position: absolute; + right: 10px; + top: 0; + bottom: 0; + display: flex; + align-items: center; + gap: 3px; + pointer-events: none; + color: #999; + font-size: 10px; + transition: opacity 0.15s ease; +} + +/* Hide the hint while the input is focused (redundant cue). */ +body.search-focused .search-kbd { + opacity: 0; +} + +.search-kbd kbd, +.shortcut-flyout-list kbd, +.article kbd { + display: inline-flex; + align-items: center; + justify-content: center; + padding: 3px 5px 1px; + font-family: var(--font-body); + font-size: 10px; + font-weight: 500; + line-height: 1; + color: var(--kbd-text); + background: var(--kbd-bg); + border: 1px solid var(--kbd-border); + border-bottom-width: 1.5px; + border-radius: 3px; +} + +/* Bigger pills: the '?' help flyout and the keyboard keys in prose / + tables on the "About this site" page (body content, not a hint). */ +.shortcut-flyout-list kbd, +.article kbd { + padding: 4px 7px; + font-size: 12px; + vertical-align: baseline; +} + +/* ====================================================================== */ +/* Search results page */ +/* ====================================================================== */ + +/* ---- Big search input (top of page) -------------------------------- */ + +.search-page-form { + position: relative; + display: flex; + align-items: stretch; + margin: 0 0 1.5em; + border: 1px solid var(--border); + border-radius: 4px; + background: var(--bg); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.04); + transition: border-color 0.15s, box-shadow 0.15s; +} + +.search-page-form:focus-within { + border-color: var(--accent); + box-shadow: 0 0 0 3px var(--accent-ring); +} + +.search-page-icon { + align-self: center; + padding: 0 0.4em 0 0.9em; + color: var(--text-muted); + font-size: 0.95rem; +} + +.search-page-input { + flex: 1; + min-width: 0; + padding: 0.7em 0.4em; + font-size: 1.15rem; + font-family: var(--font-body); + color: var(--text); + background: transparent; + border: none; + outline: none; + box-sizing: border-box; +} + +.search-page-btn { + flex-shrink: 0; + padding: 0 1.4em; + font-size: 0.95rem; + font-weight: 500; + color: var(--text-muted); + background: transparent; + border: none; + border-left: 1px solid var(--border); + cursor: pointer; + transition: color 0.15s, background 0.15s; +} + +.search-page-btn:hover { + color: var(--accent); + background: var(--bg-key); +} + +/* ---- Result cards -------------------------------------------------- */ + +/* "Search finished, found N pages..." strapline. */ +p.search-summary { + font-size: 0.9em; + font-weight: 500; + margin: 0 0 1em 0; + color: var(--text-muted); +} + +ul.search { + list-style: none; + margin: 0; + padding: 8px 0; + display: flex; + flex-direction: column; + gap: 8px; +} + +ul.search > li { + list-style: none; + margin: 0; + padding: 12px 16px; + display: flex; + flex-wrap: wrap; + align-items: flex-start; + column-gap: 6px; + line-height: normal; + border: 1px solid var(--border); + border-radius: 6px; + background: var(--bg); + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.04); +} + +[data-theme="dark"] ul.search > li { + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.35); +} + +/* Compact card: link-only result with no description. */ +ul.search > li:not(:has(> span)):not(:has(> p.context)) { + padding: 8px 16px; +} + +ul.search > li > a, +ul.search > li > a:visited, +ul.search > li > a *, +ul.search > li > a:visited * { + flex: 1; + font-weight: bold; + color: var(--accent); + text-decoration: none; +} + +ul.search > li > a:hover { + text-decoration: underline; +} + +/* Description chip e.g. "(Python class, in API reference)" */ +ul.search > li > span:not(.highlighted) { + font-size: 0.75em; + font-weight: 500; + color: var(--text-muted); + background: var(--bg-key); + border: 1px solid var(--border); + border-radius: 10px; + padding: 0.1em 0.55em; + align-self: center; + opacity: 0.85; + white-space: nowrap; +} + +/* Context paragraph (snippet under the title), aligned with the + icon column. */ +ul.search > li > p.context { + flex-basis: 100%; + margin: 0 0 0 28px; + font-size: 0.9em; + color: var(--text-muted); + line-height: 1.5; + display: -webkit-box; + -webkit-line-clamp: 2; + line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; +} + +ul.search .highlighted, +ul.search > li > p.context .highlighted { + background: var(--bg-search-highlight); + padding: 0 2px; + border-radius: 2px; +} + +/* Active row when navigating with ArrowUp / ArrowDown. */ +ul.search > li.search-result-active { + background: var(--bg-key); + outline: none; +} + +/* ---- Result icons via FontAwesome ::before ------------------------- */ + +/* Icon column. Page-specific glyph comes from --doc-icon (set per page + in doc-icons.css, shared with the sidebar); the kind fallback below + covers results whose target page has no icon. */ +ul.search > li::before { + content: var(--doc-icon, "\f0f6"); /* generic doc */ + font-family: var(--doc-icon-font, "Font Awesome 7 Free"); + font-weight: var(--doc-icon-weight, 900); + font-size: 16px; + flex-shrink: 0; + margin-top: 4px; + margin-right: 2px; + width: 1.2em; + text-align: center; + color: var(--text-muted); +} + +/* Fallbacks by Sphinx SearchResultKind, used only when the target page + has no --doc-icon. */ +ul.search > li.kind-title::before { + content: var(--doc-icon, "\f0c1"); +} /* link */ + +ul.search > li.kind-object::before { + content: var(--doc-icon, "\f085"); +} /* cogs */ + +ul.search > li.kind-text::before { + content: var(--doc-icon, "\f0f6"); +} /* doc text */ + +ul.search > li.kind-index::before { + content: var(--doc-icon, "\f097"); +} /* bookmark */ + +/* Blog results get a "Blog >" breadcrumb prefix on the link itself. */ +ul.search > li:has(> a[href^="blog/"]) > a::before, +ul.search > li:has(> a[href*="/blog/"]) > a::before { + content: "Blog > "; +} diff --git a/docs/_static/css/tables.css b/docs/_static/css/tables.css new file mode 100644 index 0000000000..3dca2817dd --- /dev/null +++ b/docs/_static/css/tables.css @@ -0,0 +1,98 @@ +/* + * psutil-sphinx-theme: HTML tables + version directives. + */ + +/* ---- Tables -------------------------------------------------------- */ + +.article table.docutils { + border-collapse: collapse; + width: 100%; + margin: 1em 0; + table-layout: fixed; +} + +.article table.docutils th, +.article table.docutils td { + padding: 6px 10px; + border: 1px solid var(--table-border); + vertical-align: top; + text-align: left; + white-space: normal; + word-wrap: break-word; + overflow-wrap: break-word; +} + +.article table.docutils th { + background: var(--table-header-bg); + color: var(--table-header-text); + font-weight: 600; + text-align: left; +} + +.article table.docutils tbody tr:nth-child(odd) td { + background: var(--table-row-odd); +} + +.article table.docutils tbody tr:nth-child(even) td { + background: var(--table-row-even); +} + +/* Cell paragraphs shouldn't add bottom margin. */ +.article table.docutils th p, +.article table.docutils td p { + margin: 0; +} + +/* "wide-table" opt-out: size columns to content. */ +.article table.wide-table { + table-layout: auto; +} + +/* Narrow screens: let tables scroll horizontally. */ +@media (max-width: 1024px) { + .article table.docutils { + display: block; + overflow-x: auto; + table-layout: auto; + max-width: 100%; + } +} + +/* ---- Version directives -------------------------------------------- */ + +div.versionadded, +div.versionchanged, +div.deprecated { + border-left: 3px solid; + padding: 0 1rem; + margin: 1em 0; +} + +div.versionadded p, +div.versionchanged p, +div.deprecated p { + margin: 0; +} + +div.versionadded { + border-left-color: var(--versionadded-border); +} +div.versionchanged { + border-left-color: var(--versionchanged-border); +} +div.deprecated { + border-left-color: var(--deprecated-border); +} + +div.versionadded .versionmodified { + color: var(--versionadded-color); + font-weight: 600; +} +div.versionchanged .versionmodified { + color: var(--versionchanged-color); + font-weight: 600; +} +div.deprecated .versionmodified { + color: var(--deprecated-color); + font-weight: 600; +} diff --git a/docs/_static/css/topbar.css b/docs/_static/css/topbar.css new file mode 100644 index 0000000000..cf35fea0bb --- /dev/null +++ b/docs/_static/css/topbar.css @@ -0,0 +1,251 @@ +/* ---- Hamburger (mobile only) ---------------------------------------- */ + +.topbar-hamburger { + display: none; + align-items: center; + justify-content: center; + width: 38px; + height: 38px; + margin-right: 4px; + border: none; + border-radius: 6px; + background: transparent; + color: var(--sidebar-fg); + font-size: 1.25rem; + cursor: pointer; +} + +@media (max-width: 1024px) { + .topbar-hamburger { + display: inline-flex; + } +} + +/* ---- Container ------------------------------------------------------- */ + +.topbar { + background: + linear-gradient( + to bottom, + rgba(255, 255, 255, 0.1) 0%, + rgba(255, 255, 255, 0) 100% + ), + var(--bg-topbar); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.35); + border-bottom: 1px solid rgba(255, 255, 255, 0.1); + color: var(--sidebar-fg); +} + +.topbar-inner { + display: flex; + align-items: center; + justify-content: space-between; + height: 100%; + max-width: var(--layout-width); + margin: 0 auto; + padding: 0 2.5rem 0 0; +} + +/* ---- Left: logo block (same width as the left sidebar) -------------- */ + +.topbar .topbar-logo, +.topbar .topbar-logo:visited { + display: flex; + align-items: center; + height: 100%; + width: var(--left-sidebar-width); + /* Match the left sidebar's item indentation (1.618em) so the + wordmark sits directly above where the toctree items start. */ + padding-left: 1.618em; + gap: 10px; + text-decoration: none; + color: #ffffff; + font-weight: 500; + font-size: 1.1rem; + letter-spacing: 0.01em; +} + +.topbar-logo span { + padding-top: 6px; + line-height: 1; +} + +.topbar-logo img { + height: 30px; + width: auto; + display: block; +} + +/* ---- Right: actions cluster ----------------------------------------- */ + +.topbar-actions { + display: flex; + align-items: center; + gap: 10px; +} + +/* Visual gap between the "preferences/nav" group (Blog, theme) and the + "repo facts" group (GitHub, version). */ +.topbar-btn + .topbar-link { + margin-left: 6px; +} + +/* Square icon button base (theme-toggle button + GitHub link). FA + icons inside scale via font-size. */ +.topbar-link, +.topbar-btn { + display: inline-flex; + align-items: center; + justify-content: center; + width: 32px; + height: 32px; + border-radius: 6px; + border: none; + background: transparent; + cursor: pointer; + color: var(--sidebar-fg); + font-size: 1.3rem; + text-decoration: none; + transition: background 0.15s, color 0.15s; +} + +/* Blog link reads as a text label, not an icon button. */ +.topbar-text { + display: inline-flex; + align-items: center; + height: 32px; + padding: 0 10px; + font-size: 1.05rem; + font-weight: 600; + color: var(--sidebar-fg); + text-decoration: none; + border-radius: 6px; + transition: background 0.15s, color 0.15s; +} + +/* Shared hover for every actionable element in the topbar. */ +.topbar-hamburger:hover, +.topbar-link:hover, +.topbar-btn:hover, +.topbar-text:hover { + background: var(--bg-hover-on-dark); + color: #ffffff; +} + +.topbar a:hover { + text-decoration: none; +} + +/* GitHub: icon + star count, expands beyond the 32px square. */ +.topbar-github { + width: auto; + padding: 0 8px 0 6px; + gap: 6px; +} + +.topbar-stars { + display: inline-flex; + align-items: center; + gap: 3px; + font-size: 0.9rem; + font-weight: 600; + line-height: 1; + font-variant-numeric: tabular-nums; + color: var(--sidebar-fg); +} + +/* FontAwesome star, matching the old site exactly. */ +.topbar-stars::before { + content: "\f005"; + font-family: "Font Awesome 7 Free"; + font-weight: 900; + font-size: 0.85em; + opacity: 0.85; +} + +.topbar-stars:empty { + display: none; +} + +/* ---- Mobile responsive ---------------------------------------------- */ + +@media (max-width: 1024px) { + .topbar-logo, + .topbar-logo:visited { + width: auto; + padding-left: 0; + } + + .topbar-inner { + padding: 0 8px; + justify-content: flex-start; + } + + .topbar-hamburger { + margin-right: 0; + } + + .topbar-actions { + gap: 0; + margin-left: auto; + } + + .topbar-btn + .topbar-link { + margin-left: 0; + } + + .topbar-text { + padding: 0 8px; + } + + /* Drop the stars count once the hamburger is in play; the + icon-only GitHub link stays. */ + .topbar-stars { + display: none; + } + + .topbar-github { + padding: 0 6px; + } + + .topbar-link, + .topbar-btn, + .topbar-hamburger { + width: 38px; + } +} + +@media (max-width: 600px) { + .topbar-text { + font-size: 0.95rem; + } +} + +@media (max-width: 620px) { + .topbar-actions .topbar-text { + display: none; + } +} + +@media (max-width: 360px) { + .topbar-logo span { + display: none; + } +} + +@media (pointer: coarse) { + .topbar-link, + .topbar-btn, + .topbar-hamburger { + width: 44px; + height: 44px; + } + + .topbar-text { + height: 44px; + } +} + +.topbar-toc-toggle { + display: none; +} diff --git a/docs/_static/css/typography.css b/docs/_static/css/typography.css new file mode 100644 index 0000000000..e20e18e771 --- /dev/null +++ b/docs/_static/css/typography.css @@ -0,0 +1,240 @@ +/* ---- Body ---------------------------------------------------------- */ + +body { + font-family: var(--font-body); + /* fix flashing */ + background: var(--bg); + color: var(--text); + font-size: 16px; + line-height: 1.65; +} + +/* ---- Headings (article only) -------------------------------------- */ + +.article h1, +.article h2, +.article h3, +.article h4 { + color: var(--headings); + font-family: var(--font-body); + margin-top: 14px; + margin-bottom: 8px; +} + +.article h1 { + font-size: 230%; + font-weight: 700; + line-height: 1.2; + letter-spacing: -0.015em; + padding: 4px 0 2px; + margin-bottom: 0.8rem; +} + +.article h1 a, +.article h2 a, +.article h3 a, +.article h4 a, +.article h1 a:visited, +.article h2 a:visited, +.article h3 a:visited, +.article h4 a:visited { + color: inherit; + text-decoration: none; +} + +/* Anchor link (¶) replaced with a FontAwesome chain-link icon, shown + only on heading hover. Same pattern as the old RTD theme. */ +.article a.headerlink, +.article a.headerlink:visited, +.article a.headerlink:hover { + visibility: visible; /* override basic.css default */ + margin-left: 0.6rem; + padding-left: 0; + color: var(--text-muted); + font-weight: normal; + font-size: 0; + text-decoration: none; + opacity: 0; + transition: opacity 0.15s ease; +} + +.article a.headerlink::after, +.article a.headerlink:hover::after { + font-family: "Font Awesome 7 Free"; + font-weight: 900; + content: "\f0c1"; + font-size: 0.85rem; + color: var(--text-muted); +} + +.article h1:hover a.headerlink, +.article h2:hover a.headerlink, +.article h3:hover a.headerlink, +.article h4:hover a.headerlink, +.article h5:hover a.headerlink, +.article h6:hover a.headerlink, +.article dt:hover a.headerlink { + opacity: 0.7; +} + +.article a.headerlink:hover, +.article a.headerlink:focus-visible { + opacity: 1; +} + +.article a.headerlink:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 2px; +} + +.article h2 { + font-size: 150%; + font-weight: 600; + line-height: 1.3; + padding: 2px 0; + margin-top: 1.4rem; + margin-bottom: 0.5rem; +} + +.article h3 { + font-size: 125%; + font-weight: 600; + line-height: 1.35; + padding: 2px 0; + margin-top: 1rem; +} + +.article h4 { + font-size: 115%; + font-weight: 500; + padding: 2px 0; +} + +/* ---- Paragraphs and lists ----------------------------------------- */ + +.article p { + margin-bottom: 16px; +} + +.article ul, +.article ol { + margin: 0 0 1em; + padding-left: 1.5em; +} + +.article li { + margin-bottom: 0.3em; +} + +/* ---- Links (prose only) ------------------------------------------- */ + +.article a { + color: var(--accent); + text-decoration: none; +} + +.article a:hover { + text-decoration: underline; +} + +.article a.reference:has(code.xref) { + color: var(--links-api); +} + +.article a.reference:has(code.xref):hover { + text-decoration: none; +} + +/* External links get a small "↗" FontAwesome icon after the text. */ +.article a.reference.external:not(:has(img)):not([href^="#"])::after { + content: "\f08e"; + font-family: "Font Awesome 7 Free"; + font-weight: 900; + font-size: 0.4em; + margin-left: 0.3em; + vertical-align: super; + opacity: 0.6; + display: inline-block; +} + +/* Pull the icon flush against trailing inline code to avoid a gap. */ +.article a.reference.external:has(> code):not(:has(img))::after, +.article a.reference.external:has(> span > code):not(:has(img))::after { + margin-left: 0; +} + +.article a.reference code, +.article a.reference tt { + color: inherit; + background: none; + font-size: 90%; + font-weight: normal; + text-decoration: underline; + text-decoration-color: var(--links-api-underline); + text-decoration-thickness: 1px; + text-underline-offset: 4px; +} + +/* ---- Inline code (`literals`) ------------------------------------- */ + +code { + font-family: var(--font-mono); + font-size: 92%; + font-weight: normal; + color: var(--inline-code-text); + background: var(--inline-code-bg); + padding: 0.1em 0.35em; + border-radius: 4px; + border: none; + overflow-wrap: break-word; +} + +pre code { + background: transparent; + padding: 0; + border-radius: 0; + font-size: inherit; + color: inherit; +} + +.article a.reference.external { + overflow-wrap: break-word; +} + +.article code.docutils.literal span.pre, +.article dl:not(.docutils) > dt span.pre { + white-space: normal; +} + +.article dl:not(.docutils) > dt { + overflow-wrap: break-word; +} + +/* ---- Blockquotes --------------------------------------------------- */ + +blockquote { + margin: 1em 0; + padding: 0 1em; + border-left: 3px solid var(--border); + color: var(--text-secondary); +} + +/* ---- Horizontal rule ----------------------------------------------- */ + +hr { + border: 0; + border-top: 1px solid var(--border); + margin: 2em 0; +} + +/* ---- In-content TOC (`.. contents::`) ----------------------------- */ + +/* basic.css wraps these in a 1px #ccc border with padding; drop the + border so the list reads as native prose. */ +.article aside.topic, +.article div.topic, +.article nav.contents { + border: 0; + padding: 0; + background: transparent; +} diff --git a/docs/_static/css/versions.css b/docs/_static/css/versions.css new file mode 100644 index 0000000000..922994a19f --- /dev/null +++ b/docs/_static/css/versions.css @@ -0,0 +1,141 @@ +/* Version selector in the topbar. */ + +.topbar-versions { + position: relative; + user-select: none; +} + +.topbar-versions > summary { + display: inline-flex; + align-items: center; + gap: 5px; + height: 32px; + padding: 0 8px; + border-radius: 6px; + color: var(--sidebar-fg); + font-size: 0.95rem; + font-weight: 600; + line-height: 1; + cursor: pointer; + list-style: none; + transition: background 0.15s, color 0.15s; +} + +.topbar-versions > summary::-webkit-details-marker { + display: none; +} + +.topbar-versions > summary::before { + content: "\f02b"; + font-family: "Font Awesome 7 Free"; + font-weight: 900; + font-size: 0.9em; + opacity: 0.85; +} + +.topbar-versions > summary:hover, +.topbar-versions[open] > summary { + background: var(--bg-hover-on-dark); + color: #ffffff; +} + +.topbar-versions > summary::after { + content: "\f077"; + font-family: "Font Awesome 7 Free"; + font-weight: 900; + font-size: 0.7em; + opacity: 0.9; + transform: rotate(180deg); + transition: transform 0.15s; +} + +.topbar-versions[open] > summary::after { + transform: rotate(0deg); +} + +.topbar-versions-menu { + position: absolute; + top: calc(100% + 6px); + right: 0; + z-index: 200; + min-width: 168px; + padding: 6px 0; + border: 1px solid var(--border); + border-radius: 8px; + background: var(--bg); + box-shadow: + 0 0 6px rgba(0, 0, 0, 0.06), + 0 2px 8px rgba(0, 0, 0, 0.1), + 0 8px 24px rgba(0, 0, 0, 0.16); +} + +[data-theme="dark"] .topbar-versions-menu { + border-color: rgba(255, 255, 255, 0.14); + background: var(--surface-raised); + box-shadow: 0 2px 10px rgba(0, 0, 0, 0.5), 0 10px 28px rgba(0, 0, 0, 0.6); +} + +[data-theme="dark"] .topbar-versions-sep { + border-top-color: rgba(255, 255, 255, 0.1); +} + +.topbar-versions-sep { + height: 0; + margin: 5px 0; + border: 0; + border-top: 1px solid var(--border); +} + +.topbar-versions-menu .topbar-versions-item, +.topbar-versions-menu .topbar-versions-item:visited { + display: flex; + align-items: baseline; + gap: 7px; + padding: 5px 14px; + color: var(--text); + font-size: 0.88rem; + text-decoration: none; +} + +.topbar-versions-name { + font-weight: 600; +} + +.topbar-versions-item:hover { + background: var(--bg-hover); + text-decoration: none; +} + +.topbar-versions-note { + margin-left: auto; + padding-left: 18px; + color: var(--text-muted); + font-size: 0.76rem; +} + +.topbar-versions-menu { + max-width: calc(100vw - 24px); +} + +@media (max-width: 1024px) { + .topbar-versions > summary { + padding: 0 8px; + } +} + +@media (max-width: 600px) { + .topbar-versions > summary::before { + display: none; + } +} + +@media (pointer: coarse) { + .topbar-versions > summary { + height: 44px; + } + + .topbar-versions-menu .topbar-versions-item { + padding-top: 9px; + padding-bottom: 9px; + } +} diff --git a/docs/_static/favicon.ico b/docs/_static/favicon.ico deleted file mode 100644 index c9efc5844a..0000000000 Binary files a/docs/_static/favicon.ico and /dev/null differ diff --git a/docs/_static/fonts/README.txt b/docs/_static/fonts/README.txt new file mode 100644 index 0000000000..748ac393e2 --- /dev/null +++ b/docs/_static/fonts/README.txt @@ -0,0 +1,23 @@ +Self-hosted web fonts (latin subset), wired up in ../css/fonts.css. + +All three families are licensed under the SIL Open Font License 1.1. +The full license text ships with each upstream project (linked below); +see also https://openfontlicense.org. + +- Inter + Copyright (c) The Inter Project Authors. + https://github.com/rsms/inter + +- JetBrains Mono + Copyright (c) The JetBrains Mono Project Authors. + https://github.com/JetBrains/JetBrainsMono + +- Merriweather + Copyright (c) The Merriweather Project Authors. + https://github.com/SorkinType/Merriweather + +Files were generated from the Google Fonts css2 API (latin unicode-range +only), then trimmed of unused OpenType features with pyftsubset: + + pyftsubset FONT.woff2 --unicodes='*' --layout-features='kern,liga,tnum' \ + --no-hinting --flavor=woff2 --output-file=FONT.woff2 diff --git a/docs/_static/fonts/fa-brands-subset.woff2 b/docs/_static/fonts/fa-brands-subset.woff2 new file mode 100644 index 0000000000..de928e846f Binary files /dev/null and b/docs/_static/fonts/fa-brands-subset.woff2 differ diff --git a/docs/_static/fonts/fa-regular-subset.woff2 b/docs/_static/fonts/fa-regular-subset.woff2 new file mode 100644 index 0000000000..e5fa8b9b4d Binary files /dev/null and b/docs/_static/fonts/fa-regular-subset.woff2 differ diff --git a/docs/_static/fonts/fa-solid-subset.woff2 b/docs/_static/fonts/fa-solid-subset.woff2 new file mode 100644 index 0000000000..73e4b7d2fc Binary files /dev/null and b/docs/_static/fonts/fa-solid-subset.woff2 differ diff --git a/docs/_static/fonts/inter-400.woff2 b/docs/_static/fonts/inter-400.woff2 new file mode 100644 index 0000000000..03cc794e7b Binary files /dev/null and b/docs/_static/fonts/inter-400.woff2 differ diff --git a/docs/_static/fonts/inter-500.woff2 b/docs/_static/fonts/inter-500.woff2 new file mode 100644 index 0000000000..03cc794e7b Binary files /dev/null and b/docs/_static/fonts/inter-500.woff2 differ diff --git a/docs/_static/fonts/inter-600.woff2 b/docs/_static/fonts/inter-600.woff2 new file mode 100644 index 0000000000..03cc794e7b Binary files /dev/null and b/docs/_static/fonts/inter-600.woff2 differ diff --git a/docs/_static/fonts/inter-700.woff2 b/docs/_static/fonts/inter-700.woff2 new file mode 100644 index 0000000000..03cc794e7b Binary files /dev/null and b/docs/_static/fonts/inter-700.woff2 differ diff --git a/docs/_static/fonts/jetbrains-mono-400.woff2 b/docs/_static/fonts/jetbrains-mono-400.woff2 new file mode 100644 index 0000000000..28f3f7322a Binary files /dev/null and b/docs/_static/fonts/jetbrains-mono-400.woff2 differ diff --git a/docs/_static/fonts/jetbrains-mono-600.woff2 b/docs/_static/fonts/jetbrains-mono-600.woff2 new file mode 100644 index 0000000000..28f3f7322a Binary files /dev/null and b/docs/_static/fonts/jetbrains-mono-600.woff2 differ diff --git a/docs/_static/fonts/merriweather-400-italic.woff2 b/docs/_static/fonts/merriweather-400-italic.woff2 new file mode 100644 index 0000000000..b2d66cb8dc Binary files /dev/null and b/docs/_static/fonts/merriweather-400-italic.woff2 differ diff --git a/docs/_static/fonts/merriweather-400.woff2 b/docs/_static/fonts/merriweather-400.woff2 new file mode 100644 index 0000000000..a3298284c5 Binary files /dev/null and b/docs/_static/fonts/merriweather-400.woff2 differ diff --git a/docs/_static/fonts/merriweather-700.woff2 b/docs/_static/fonts/merriweather-700.woff2 new file mode 100644 index 0000000000..a3298284c5 Binary files /dev/null and b/docs/_static/fonts/merriweather-700.woff2 differ diff --git a/docs/_static/images/about/backtotop.png b/docs/_static/images/about/backtotop.png new file mode 100644 index 0000000000..3d006afda6 Binary files /dev/null and b/docs/_static/images/about/backtotop.png differ diff --git a/docs/_static/images/about/clickable.png b/docs/_static/images/about/clickable.png new file mode 100644 index 0000000000..1edb323020 Binary files /dev/null and b/docs/_static/images/about/clickable.png differ diff --git a/docs/_static/images/about/copy.png b/docs/_static/images/about/copy.png new file mode 100644 index 0000000000..cdfa4fd5fd Binary files /dev/null and b/docs/_static/images/about/copy.png differ diff --git a/docs/_static/images/about/copypage.png b/docs/_static/images/about/copypage.png new file mode 100644 index 0000000000..43e111a94c Binary files /dev/null and b/docs/_static/images/about/copypage.png differ diff --git a/docs/_static/images/about/darkmode-dark.png b/docs/_static/images/about/darkmode-dark.png new file mode 100644 index 0000000000..0778c80da2 Binary files /dev/null and b/docs/_static/images/about/darkmode-dark.png differ diff --git a/docs/_static/images/about/darkmode-light.png b/docs/_static/images/about/darkmode-light.png new file mode 100644 index 0000000000..8074c42a1c Binary files /dev/null and b/docs/_static/images/about/darkmode-light.png differ diff --git a/docs/_static/images/about/mobile.png b/docs/_static/images/about/mobile.png new file mode 100644 index 0000000000..dfd46485d3 Binary files /dev/null and b/docs/_static/images/about/mobile.png differ diff --git a/docs/_static/images/about/search.png b/docs/_static/images/about/search.png new file mode 100644 index 0000000000..60e05a4f0f Binary files /dev/null and b/docs/_static/images/about/search.png differ diff --git a/docs/_static/images/about/toc.png b/docs/_static/images/about/toc.png new file mode 100644 index 0000000000..9c812a2d8a Binary files /dev/null and b/docs/_static/images/about/toc.png differ diff --git a/docs/_static/images/about/versions.png b/docs/_static/images/about/versions.png new file mode 100644 index 0000000000..d4e8992ded Binary files /dev/null and b/docs/_static/images/about/versions.png differ diff --git a/docs/_static/images/favicon.svg b/docs/_static/images/favicon.svg new file mode 100644 index 0000000000..65a8a36c53 --- /dev/null +++ b/docs/_static/images/favicon.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/docs/_static/images/icon-cpu.svg b/docs/_static/images/icon-cpu.svg new file mode 100644 index 0000000000..59a2c5e107 --- /dev/null +++ b/docs/_static/images/icon-cpu.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/docs/_static/images/icon-disks.svg b/docs/_static/images/icon-disks.svg new file mode 100644 index 0000000000..9eacdf03cc --- /dev/null +++ b/docs/_static/images/icon-disks.svg @@ -0,0 +1,7 @@ + + + + + + diff --git a/docs/_static/images/icon-memory.svg b/docs/_static/images/icon-memory.svg new file mode 100644 index 0000000000..e499ac5aa1 --- /dev/null +++ b/docs/_static/images/icon-memory.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/docs/_static/images/icon-network.svg b/docs/_static/images/icon-network.svg new file mode 100644 index 0000000000..4abfa54c31 --- /dev/null +++ b/docs/_static/images/icon-network.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/docs/_static/images/icon-processes.svg b/docs/_static/images/icon-processes.svg new file mode 100644 index 0000000000..90b94bf577 --- /dev/null +++ b/docs/_static/images/icon-processes.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/docs/_static/images/icon-sensors.svg b/docs/_static/images/icon-sensors.svg new file mode 100644 index 0000000000..2b1ccb0eb3 --- /dev/null +++ b/docs/_static/images/icon-sensors.svg @@ -0,0 +1,7 @@ + + + + + + diff --git a/docs/_static/images/logo-apivoid.svg b/docs/_static/images/logo-apivoid.svg new file mode 100644 index 0000000000..49ebb25cd8 --- /dev/null +++ b/docs/_static/images/logo-apivoid.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/docs/_static/images/logo-psutil-readme.svg b/docs/_static/images/logo-psutil-readme.svg new file mode 100644 index 0000000000..45ac91842f --- /dev/null +++ b/docs/_static/images/logo-psutil-readme.svg @@ -0,0 +1,20 @@ + + + + + + + + psutil + diff --git a/docs/_static/images/logo-psutil.png b/docs/_static/images/logo-psutil.png new file mode 100644 index 0000000000..d02ab0af4f Binary files /dev/null and b/docs/_static/images/logo-psutil.png differ diff --git a/docs/_static/images/logo-psutil.svg b/docs/_static/images/logo-psutil.svg new file mode 100644 index 0000000000..245e606bca --- /dev/null +++ b/docs/_static/images/logo-psutil.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/docs/_static/images/logo-sansec.svg b/docs/_static/images/logo-sansec.svg new file mode 100644 index 0000000000..9fa30a4e0e --- /dev/null +++ b/docs/_static/images/logo-sansec.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/_static/images/logo-tidelift.svg b/docs/_static/images/logo-tidelift.svg new file mode 100644 index 0000000000..448dd7a081 --- /dev/null +++ b/docs/_static/images/logo-tidelift.svg @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + diff --git a/docs/_static/images/rss.svg b/docs/_static/images/rss.svg new file mode 100644 index 0000000000..cdb14648b1 --- /dev/null +++ b/docs/_static/images/rss.svg @@ -0,0 +1 @@ + diff --git a/docs/_static/js/back-to-top.js b/docs/_static/js/back-to-top.js new file mode 100644 index 0000000000..5eaf3ea0a6 --- /dev/null +++ b/docs/_static/js/back-to-top.js @@ -0,0 +1,52 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Floating "back to top" button. Appears when the user has scrolled +// down and is moving back up; hidden again at the top of the page. + +(function () { + const btn = document.createElement("button"); + btn.type = "button"; + btn.className = "back-to-top"; + btn.setAttribute("aria-label", "Back to top"); + btn.title = "Back to top"; + // Hidden at rest via opacity, which doesn't remove it from the tab + // order; keep it unfocusable until the scroll handler shows it. + btn.tabIndex = -1; + btn.innerHTML = + ''; + // Faster than the browser's default smooth scroll (which can take + // ~1s on long pages). Custom 300ms ease-out animation. + btn.addEventListener("click", () => { + if (window.matchMedia("(prefers-reduced-motion: reduce)").matches) { + window.scrollTo(0, 0); + return; + } + const start = window.scrollY; + const t0 = performance.now(); + function step(now) { + const t = Math.min((now - t0) / 300, 1); + const eased = 1 - Math.pow(1 - t, 3); + window.scrollTo(0, start * (1 - eased)); + if (t < 1) { + requestAnimationFrame(step); + } + } + requestAnimationFrame(step); + }); + document.body.appendChild(btn); + + // Show only while scrolling UP (and past the threshold); hide + // again on any downward scroll. + const SHOW_AFTER_PX = 400; + let lastY = window.scrollY; + window.addEventListener("scroll", () => { + const y = window.scrollY; + const goingUp = y < lastY; + const visible = goingUp && y > SHOW_AFTER_PX; + btn.classList.toggle("is-visible", visible); + btn.tabIndex = visible ? 0 : -1; + lastY = y; + }, { passive: true }); +})(); diff --git a/docs/_static/js/banner.js b/docs/_static/js/banner.js new file mode 100644 index 0000000000..f6062367b1 --- /dev/null +++ b/docs/_static/js/banner.js @@ -0,0 +1,47 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Notice above the topbar. Archived releases inject the same markup +// from build_versions.py, and those pages have no .header-stack. + +(function () { + const KEY = "psutil-banner"; + const root = document.documentElement; + const banner = document.querySelector(".site-banner"); + if (!banner) { + return; + } + + // The banner is part of the fixed header and its text wraps, so + // the offset below it has to be measured, not hardcoded. + const stack = banner.closest(".header-stack"); + + function syncHeight() { + if (stack) { + root.style.setProperty( + "--header-height", + stack.offsetHeight + "px", + ); + } + } + + const close = banner.querySelector(".site-banner-close"); + if (close) { + close.addEventListener("click", () => { + root.classList.add("site-banner-dismissed"); + try { + localStorage.setItem(KEY, banner.dataset.bannerId || ""); + } + catch (err) { + console.warn("banner: " + err.message); + } + syncHeight(); + }); + } + + if (stack) { + new ResizeObserver(syncHeight).observe(stack); + syncHeight(); + } +})(); diff --git a/docs/_static/js/blog-comment-counts.js b/docs/_static/js/blog-comment-counts.js new file mode 100644 index 0000000000..4b3e903e99 --- /dev/null +++ b/docs/_static/js/blog-comment-counts.js @@ -0,0 +1,53 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Comment counts on the blog listing. + +(function () { + const list = document.querySelector(".blog-cards[data-comments-repo]"); + if (!list) { + return; + } + + const repo = list.dataset.commentsRepo; + const url = "https://api.github.com/repos/" + repo + + "/discussions?per_page=100"; + + function render(discussions) { + const counts = new Map(); + for (const disc of discussions) { + counts.set(disc.title, disc.comments); + } + for (const card of list.querySelectorAll(".blog-card[data-docname]")) { + const num = counts.get(card.dataset.docname); + if (!num) { + continue; + } + const meta = card.querySelector(".blog-card-meta"); + if (!meta) { + continue; + } + const span = document.createElement("span"); + span.className = "blog-card-comments"; + span.textContent = "\u{1F4AC} " + num; + span.setAttribute( + "aria-label", + num === 1 ? "1 comment" : num + " comments", + ); + meta.appendChild(span); + } + } + + fetch(url, { headers: { Accept: "application/vnd.github+json" } }) + .then((resp) => { + if (!resp.ok) { + throw new Error("HTTP " + resp.status); + } + return resp.json(); + }) + .then(render) + .catch((err) => { + console.warn("could not load blog comment counts:", err); + }); +})(); diff --git a/docs/_static/js/copy-page.js b/docs/_static/js/copy-page.js new file mode 100644 index 0000000000..16d3ac807e --- /dev/null +++ b/docs/_static/js/copy-page.js @@ -0,0 +1,86 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// "Copy page" button next to the h1: copies the page's RsT source, which +// Sphinx publishes under _sources/. + +(function () { + const RESET_DELAY_MS = 2000; + + const ICONS = ` + +`; + + const SKIP = ["index", "404", "blog"]; + + const page = document.body.dataset.page; + const root = document.documentElement.dataset.content_root; + if (!page || !root || SKIP.includes(page)) { + return; + } + const h1 = document.querySelector(".article h1"); + if (!h1) { + return; + } + + const url = root + "_sources/" + page + ".rst.txt"; + const btn = document.createElement("button"); + btn.type = "button"; + btn.className = "copy-page"; + btn.title = "Copy this page as reStructuredText"; + btn.innerHTML = '' + ICONS + "" + + '' + + 'Copy page' + + 'Copied' + + ""; + + let busy = false; + + function flash(state) { + btn.classList.add(state); + setTimeout(() => { + btn.classList.remove(state); + busy = false; + }, RESET_DELAY_MS); + } + + btn.addEventListener("click", function () { + if (busy) { + return; + } + busy = true; + fetch(url) + .then((resp) => { + if (!resp.ok) { + throw new Error(url + " returned " + resp.status); + } + return resp.text(); + }) + .then((text) => navigator.clipboard.writeText(text)) + .then(() => flash("copied")) + .catch((err) => { + console.warn("copy-page: " + err.message); + flash("failed"); + }); + }); + + // Only offer it when the source is actually published; generated + // pages (genindex, search, blog archives) have none. + fetch(url, { method: "HEAD" }) + .then((resp) => { + if (resp.ok) { + h1.appendChild(btn); + } + }) + .catch(() => {}); +})(); diff --git a/docs/_static/js/external-urls.js b/docs/_static/js/external-urls.js new file mode 100644 index 0000000000..e65e496b8f --- /dev/null +++ b/docs/_static/js/external-urls.js @@ -0,0 +1,12 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Open all external URLs in a new tab. + +document.querySelectorAll("a[href^='http']").forEach((a) => { + if (a.hostname !== location.hostname) { + a.target = "_blank"; + a.rel = "noopener noreferrer"; + } +}); diff --git a/docs/_static/js/giscus.js b/docs/_static/js/giscus.js new file mode 100644 index 0000000000..76edae3073 --- /dev/null +++ b/docs/_static/js/giscus.js @@ -0,0 +1,124 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Blog comments, backed by giscus + GitHub Discussions. + +(function () { + const container = document.querySelector(".giscus"); + if (!container) { + return; + } + + const ORIGIN = "https://giscus.app"; + const section = container.closest(".comments"); + const url = new URL(container.dataset.themeUrl, location.href).href; + let sheet = null; + let ready = false; + let pushed = null; + + function isDark() { + return document.documentElement.getAttribute("data-theme") === "dark"; + } + + // css/giscus.css branches on prefers-color-scheme, which inside + // giscus' iframe follows the reader's OS rather than our toggle. + // Swap the conditions for ones that are always (or never) true so + // the branch we want is the one that applies. + function themeFor(dark) { + const yes = "(min-width: 0px)"; + const no = "(min-width: 99999px)"; + const css = sheet + .replace(/\(prefers-color-scheme:\s*dark\)/g, dark ? yes : no) + .replace(/\(prefers-color-scheme:\s*light\)/g, dark ? no : yes); + return "data:text/css;base64," + btoa(css); + } + + function inject() { + const attrs = { + "data-repo": container.dataset.repo, + "data-repo-id": container.dataset.repoId, + "data-category": container.dataset.category, + "data-category-id": container.dataset.categoryId, + // Thread key is the source path, not the URL. + "data-mapping": "specific", + "data-term": container.dataset.term, + // Without this giscus matches by fuzzy title search, + // and our terms all share a "blog//" prefix. + "data-strict": "1", + "data-reactions-enabled": "1", + "data-emit-metadata": "0", + "data-input-position": "top", + "data-theme": sheet ? themeFor(isDark()) : url, + "data-lang": "en", + "data-loading": "lazy", + }; + pushed = isDark(); + + const script = document.createElement("script"); + script.src = ORIGIN + "/client.js"; + script.crossOrigin = "anonymous"; + script.async = true; + Object.keys(attrs).forEach((name) => { + script.setAttribute(name, attrs[name]); + }); + // A blocked script fires "error", not "load", so the section + // stays hidden rather than showing an empty widget. + script.addEventListener("load", () => { + if (section) { + section.removeAttribute("hidden"); + } + }); + container.appendChild(script); + } + + function push() { + if (!ready || !sheet || isDark() === pushed) { + return; + } + const frame = document.querySelector("iframe.giscus-frame"); + if (!frame || !frame.contentWindow) { + return; + } + const dark = isDark(); + frame.contentWindow.postMessage( + { giscus: { setConfig: { theme: themeFor(dark) } } }, + ORIGIN, + ); + pushed = dark; + } + + new MutationObserver(push).observe(document.documentElement, { + attributes: true, + attributeFilter: ["data-theme"], + }); + + // contentWindow exists well before giscus can receive messages, + // so wait for it to talk to us first. + window.addEventListener("message", (event) => { + if (event.origin !== ORIGIN || !event.data || !event.data.giscus) { + return; + } + ready = true; + push(); + }); + + // giscus fetches its theme with crossorigin="anonymous", which + // needs CORS headers and is blocked outright from localhost. + // Inlining it sidesteps both: this fetch is same-origin. + fetch(url) + .then((resp) => { + if (!resp.ok) { + throw new Error(resp.status); + } + return resp.text(); + }) + .then((css) => { + sheet = css; + }) + .catch((err) => { + // Fall back to the URL, fine wherever CORS allows it. + console.warn("giscus: inlining theme failed:", err); + }) + .finally(inject); +})(); diff --git a/docs/_static/js/github-meta.js b/docs/_static/js/github-meta.js new file mode 100644 index 0000000000..d782afff10 --- /dev/null +++ b/docs/_static/js/github-meta.js @@ -0,0 +1,66 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Fills in the star count from the GitHub API. Result is cached in +// sessionStorage for an hour to avoid hitting the rate limit on every +// page navigation. + +(function () { + const REPO = "giampaolo/psutil"; + const CACHE_KEY = "psutil-gh-meta"; + const CACHE_TTL_MS = 60 * 60 * 1000; + + function applyValues(stars) { + const s = document.querySelector(".topbar-stars"); + if (s && stars) { + s.textContent = stars; + } + } + + function formatStars(n) { + if (typeof n !== "number") { + return ""; + } + if (n >= 1000) { + return (n / 1000).toFixed(1) + "k"; + } + return String(n); + } + + let cached; + try { + cached = JSON.parse(sessionStorage.getItem(CACHE_KEY) || "null"); + } + catch (e) { + cached = null; + } + if (cached && Date.now() - cached.t < CACHE_TTL_MS) { + applyValues(cached.stars); + return; + } + + fetch("https://api.github.com/repos/" + REPO) + .then((r) => (r.ok ? r.json() : null)) + .then((repo) => { + // Skip caching on a rate-limit / error response so the next + // page load retries instead of showing empty for an hour. + if (!repo) { + return; + } + const stars = formatStars(repo.stargazers_count); + try { + sessionStorage.setItem( + CACHE_KEY, + JSON.stringify({ t: Date.now(), stars: stars }), + ); + } + catch (e) { + // private mode / storage disabled: skip caching + } + applyValues(stars); + }) + .catch(() => { + // offline / network error: leave the topbar placeholders + }); +})(); diff --git a/docs/_static/js/highlight-repl.js b/docs/_static/js/highlight-repl.js new file mode 100644 index 0000000000..31dcdc7155 --- /dev/null +++ b/docs/_static/js/highlight-repl.js @@ -0,0 +1,33 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Syntax-highlight REPL inside pycon output spans (class="go"). +// Pygments tokenizes >>> lines as Python, but leaves output as plain +// Generic.Output. + +document.querySelectorAll(".highlight-pycon .go").forEach((span) => { + let html = span.innerHTML; + // Highlight quoted strings (must run first, before we inject spans). + html = html.replace( + /('[^']*'|"[^"]*")/g, + '$1', + ); + // Highlight namedtuple field names (word before '='). + // The (?!") lookahead avoids matching class= in the injected span tags. + html = html.replace( + /\b([a-z_]\w*)=(?!")/g, + '$1=', + ); + // Highlight numbers after '=' or at the start of a line. + html = html.replace( + /(?<==)\d+\.?\d*|^\d+\.?\d*/gm, + '$&', + ); + span.innerHTML = html; +}); + +// Disable title showing up on hover. +document.querySelectorAll("a.sphinx-codeautolink-a[title]").forEach((a) => { + a.removeAttribute("title"); +}); diff --git a/docs/_static/js/home-install-copy.js b/docs/_static/js/home-install-copy.js new file mode 100644 index 0000000000..e70bef8cf9 --- /dev/null +++ b/docs/_static/js/home-install-copy.js @@ -0,0 +1,69 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Wire the home-page install pill: copy `pip install psutil` to the +// clipboard and flash a "Copied" hint on the button. + +(function () { + const wrapper = document.querySelector(".home-install"); + if (!wrapper) { + return; + } + const btn = wrapper.querySelector(".home-install-copy"); + const cmd = wrapper.querySelector(".home-install-cmd"); + if (!btn || !cmd) { + return; + } + + let resetTimer = null; + + function flashCopied() { + wrapper.classList.add("copied"); + const label = btn.getAttribute("aria-label") || ""; + if (!btn.dataset.origLabel) { + btn.dataset.origLabel = label; + } + btn.setAttribute("aria-label", "Copied"); + if (resetTimer) { + clearTimeout(resetTimer); + } + resetTimer = setTimeout(() => { + wrapper.classList.remove("copied"); + btn.setAttribute("aria-label", btn.dataset.origLabel); + }, 1400); + } + + function legacyCopy(text) { + const ta = document.createElement("textarea"); + ta.value = text; + ta.style.position = "fixed"; + ta.style.opacity = "0"; + document.body.appendChild(ta); + ta.select(); + let ok = false; + try { + ok = document.execCommand("copy"); + } + finally { + document.body.removeChild(ta); + } + return ok; + } + + btn.addEventListener("click", () => { + const text = cmd.textContent.trim(); + if (navigator.clipboard && navigator.clipboard.writeText) { + navigator.clipboard.writeText(text) + .then(flashCopied) + .catch(() => { + if (legacyCopy(text)) { + flashCopied(); + } + }); + } + else if (legacyCopy(text)) { + flashCopied(); + } + }); +})(); diff --git a/docs/_static/js/right-toc-toggle.js b/docs/_static/js/right-toc-toggle.js new file mode 100644 index 0000000000..2a793b89cd --- /dev/null +++ b/docs/_static/js/right-toc-toggle.js @@ -0,0 +1,51 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Hide TOC bar below 1280px resolution. + +(function () { + const btn = document.getElementById("toc-toggle"); + const toc = document.querySelector(".right-sidebar"); + if (!btn || !toc) { + return; + } + const body = document.body; + const wide = window.matchMedia("(min-width: 1280px)"); + + function close() { + body.classList.remove("toc-open"); + btn.setAttribute("aria-expanded", "false"); + } + + btn.addEventListener("click", (e) => { + e.stopPropagation(); + const open = body.classList.toggle("toc-open"); + btn.setAttribute("aria-expanded", open ? "true" : "false"); + }); + + toc.addEventListener("click", (e) => { + if (e.target.closest("a")) { + close(); + } + }); + + document.addEventListener("click", (e) => { + if (body.classList.contains("toc-open") && !toc.contains(e.target)) { + close(); + } + }); + + document.addEventListener("keydown", (e) => { + if (e.key === "Escape" && body.classList.contains("toc-open")) { + close(); + btn.focus(); + } + }); + + wide.addEventListener("change", (e) => { + if (e.matches) { + close(); + } + }); +})(); diff --git a/docs/_static/js/right-toc.js b/docs/_static/js/right-toc.js new file mode 100644 index 0000000000..384fa232b9 --- /dev/null +++ b/docs/_static/js/right-toc.js @@ -0,0 +1,245 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Right-side per-page TOC. Scroll-spy + auto-positioning so the +// current heading stays visible inside the (independently scrolling) +// TOC. Modeled after pydata-sphinx-theme. + +(function () { + "use strict"; + + var pageToc = document.querySelector(".right-sidebar"); + if (!pageToc) { + return; + } + + // Strip trailing "()" from function-name entries (just visual noise). + pageToc.querySelectorAll("a code .pre").forEach(function (el) { + if (el.textContent.endsWith("()")) { + el.textContent = el.textContent.slice(0, -2); + } + }); + + var tocLinks = Array.prototype.slice.call( + pageToc.querySelectorAll('a[href^="#"]'), + ); + + if (tocLinks.length === 0) { + return; + } + + function getHeading(tocLink) { + var href = tocLink.getAttribute("href"); + if (!href || !href.startsWith("#")) { + return null; + } + + var id = href.substring(1); + if (!id) { + return null; + } + + var decodedId; + try { + decodedId = decodeURIComponent(id); + } + catch (e) { + // Malformed %-escape in the href; ignore this entry. + return null; + } + + var target = document.getElementById(decodedId); + if (!target) { + return null; + } + + // Prefer the heading inside; fall back to target itself for + //
    autodoc anchors with no inner heading. + var heading = target.querySelector("h1, h2, h3, h4, h5, h6"); + if (heading) { + return heading; + } + return target; + } + + var headingsToTocLinks = new Map(); + + tocLinks.forEach(function (tocLink) { + var heading = getHeading(tocLink); + if (heading) { + headingsToTocLinks.set(heading, tocLink); + } + }); + + var observedHeadings = Array.from(headingsToTocLinks.keys()); + var titleHeight = 0; + + function refreshTitleHeight() { + var title = pageToc.querySelector(".right-sidebar-title"); + if (title) { + titleHeight = title.offsetHeight; + } + else { + titleHeight = 0; + } + } + + refreshTitleHeight(); + + // offsetTop / clientHeight are precomputed; getBoundingClientRect + // would force a layout flush each call. + function ensureVisibleInToc(link) { + var linkTop = link.offsetTop; + var linkBottom = linkTop + link.offsetHeight; + + var visibleTop = pageToc.scrollTop + titleHeight; + var visibleBottom = pageToc.scrollTop + pageToc.clientHeight; + + if (linkTop < visibleTop) { + pageToc.scrollTop = Math.max(0, linkTop - titleHeight); + } + else if (linkBottom > visibleBottom) { + pageToc.scrollTop = linkBottom - pageToc.clientHeight; + } + } + + var activeLink = null; + var activeLis = []; + + function activate(tocLink) { + if (tocLink === activeLink) { + return; + } + + if (activeLink) { + activeLink.removeAttribute("aria-current"); + } + + activeLis.forEach(function (li) { + li.classList.remove("scroll-current-leaf"); + }); + + activeLis = []; + activeLink = tocLink; + + if (!tocLink) { + return; + } + + tocLink.setAttribute("aria-current", "true"); + + var leaf = tocLink.parentNode && tocLink.parentNode.closest("li"); + + if (leaf) { + leaf.classList.add("scroll-current-leaf"); + activeLis.push(leaf); + } + + ensureVisibleInToc(tocLink); + } + + var observer; + var disableObserver = false; + + function temporarilyDisableObserver(ms) { + disableObserver = true; + setTimeout(function () { + disableObserver = false; + }, ms); + } + + function connectIntersectionObserver() { + if (observer) { + observer.disconnect(); + } + + refreshTitleHeight(); + + var topbar = document.querySelector(".topbar"); + var headerHeight = topbar ? topbar.offsetHeight : 0; + + // Active band: top 30% of viewport below the header. + var options = { + root: null, + rootMargin: "-" + headerHeight + "px 0px -70% 0px", + threshold: 0, + }; + + function callback(entries) { + if (disableObserver) { + return; + } + var entry = entries.filter(function (e) { + return e.isIntersecting; + }).pop(); + if (!entry) { + return; + } + var tocLink = headingsToTocLinks.get(entry.target); + if (tocLink) { + activate(tocLink); + } + } + + observer = new IntersectionObserver(callback, options); + + observedHeadings.forEach(function (h) { + observer.observe(h); + }); + } + + function debounce(fun, wait) { + var t; + + return function () { + clearTimeout(t); + t = setTimeout(fun, wait); + }; + } + + // Highlight the TOC entry whose href matches the URL hash. Used on + // page load and when navigating between headings on the same page. + function syncTocHash(hash) { + if (!hash || hash.length <= 1) { + return; + } + var link = tocLinks.find(function (l) { + return l.hash === hash; + }); + if (link) { + temporarilyDisableObserver(1000); + activate(link); + } + } + + window.addEventListener( + "resize", + debounce(connectIntersectionObserver, 300), + ); + + connectIntersectionObserver(); + + // Initial sync (in case the URL has a hash). + syncTocHash(location.hash); + + // Hash changes via in-page anchor clicks. + window.addEventListener("hashchange", function () { + syncTocHash(location.hash); + }); + + // Edge case: clicking a same-hash link doesn't fire hashchange, + // but we still want the TOC to re-sync (the user may have scrolled + // away from that section in between). + window.addEventListener("click", function (e) { + var link = e.target.closest("a"); + if ( + link && + link.hash && + link.hash === location.hash && + link.origin === location.origin + ) { + syncTocHash(link.hash); + } + }); +})(); diff --git a/docs/_static/js/search-dim.js b/docs/_static/js/search-dim.js new file mode 100644 index 0000000000..ab5ac0d180 --- /dev/null +++ b/docs/_static/js/search-dim.js @@ -0,0 +1,25 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Dim the rest of the page while the sidebar search input is focused. +// search.css handles the visual fade. Skipped on touch devices. + +(function () { + if (window.matchMedia("(pointer: coarse)").matches) { + return; + } + const input = document.getElementById("search-input"); + if (!input) { + return; + } + const overlay = document.createElement("div"); + overlay.className = "search-overlay"; + document.body.appendChild(overlay); + input.addEventListener("focus", () => { + document.body.classList.add("search-focused"); + }); + input.addEventListener("blur", () => { + document.body.classList.remove("search-focused"); + }); +})(); diff --git a/docs/_static/js/search-shortcuts.js b/docs/_static/js/search-shortcuts.js new file mode 100644 index 0000000000..4c58a2ed4c --- /dev/null +++ b/docs/_static/js/search-shortcuts.js @@ -0,0 +1,150 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Sidebar input: +// `CTRL+K` / `CMD+K`: focus / select the search input +// `ESC`: exit search +// +// Search results page: +// `Arrow Up` / `Arrow down`: navigate results +// `ENTER`: open result + +(function () { + // Disable Sphinx's built-in "/" shortcut so it doesn't compete. + // Done before the touch-device early return so it applies there too. + if (typeof DOCUMENTATION_OPTIONS !== "undefined") { + DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS = false; + } + + if (window.matchMedia("(pointer: coarse)").matches) { + return; + } + + function getInput() { + return document.getElementById("search-input"); + } + + // Re-submitting the same query on the search page is a browser + // no-op (URL doesn't change). Force a reload only in that case; + // for a real navigation, let the browser do its thing. + getInput()?.form?.addEventListener("submit", (e) => { + const form = e.currentTarget; + const action = new URL(form.action, location.href); + action.search = "?" + new URLSearchParams(new FormData(form)); + if ( + action.pathname === location.pathname && + action.search === location.search + ) { + e.preventDefault(); + location.reload(); + } + }); + + // ---- Ctrl+K + Esc on the sidebar input ------------------------- + + document.addEventListener("keydown", (e) => { + if ((e.ctrlKey || e.metaKey) && e.key === "k") { + const input = getInput(); + if (input) { + e.preventDefault(); + if (document.activeElement === input) { + input.blur(); + } + else { + input.focus(); + input.select(); + } + } + return; + } + if (e.key === "Escape") { + const input = getInput(); + if (input && document.activeElement === input) { + input.value = ""; + input.blur(); + } + } + }); + + // ---- Arrow-key navigation on the search results page ----------- + (function () { + const container = document.getElementById("search-results"); + if (!container) { + return; + } + + const pageInput = document.querySelector(".search-page-input"); + const sidebarInput = getInput(); + let activeIndex = -1; + + function getResults() { + return Array.from(container.querySelectorAll("ul.search > li")); + } + + function setActive(index) { + const results = getResults(); + if (!results.length) { + return; + } + index = Math.max(0, Math.min(index, results.length - 1)); + results.forEach((li) => { + li.classList.remove("search-result-active"); + }); + activeIndex = index; + results[index].classList.add("search-result-active"); + const link = results[index].querySelector("a"); + if (link) { + link.focus({ preventScroll: true }); + } + results[index].scrollIntoView({ block: "nearest" }); + } + + function clearActive() { + activeIndex = -1; + getResults().forEach((li) => { + li.classList.remove("search-result-active"); + }); + } + + // When Sphinx adds new
  • s, drop any prior selection. + new MutationObserver((mutations) => { + const hasNewLi = mutations.some((m) => { + return Array.from(m.addedNodes).some((n) => { + return n.tagName === "LI"; + }); + }); + if (hasNewLi) { + clearActive(); + } + }).observe(container, { childList: true, subtree: true }); + + function isSearchInput(el) { + return el && (el === sidebarInput || el === pageInput); + } + + document.addEventListener("keydown", (e) => { + if (!getResults().length) { + return; + } + if (isSearchInput(document.activeElement)) { + if (e.key !== "ArrowDown") { + return; + } + e.preventDefault(); + setActive(0); + return; + } + if (e.key === "ArrowDown") { + e.preventDefault(); + setActive(activeIndex === -1 ? 0 : activeIndex + 1); + } + else if (e.key === "ArrowUp") { + e.preventDefault(); + if (activeIndex > 0) { + setActive(activeIndex - 1); + } + } + }); + })(); +})(); diff --git a/docs/_static/js/shortcuts-help.js b/docs/_static/js/shortcuts-help.js new file mode 100644 index 0000000000..386e2d8f00 --- /dev/null +++ b/docs/_static/js/shortcuts-help.js @@ -0,0 +1,93 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Shows a helper when pressing "?" anywhere outside a form field, listing +// the site's keyboard shortcuts. Esc / click on backdrop / second "?" +// closes it. + +(function () { + const SHORTCUTS = [ + { keys: ["Shift", "D"], desc: "Toggle dark / light mode" }, + { keys: ["Ctrl", "K"], desc: "Focus search" }, + { keys: ["↑", "↓"], desc: "Navigate search results" }, + { keys: ["Enter"], desc: "Open highlighted search result" }, + { keys: ["Esc"], desc: "Close search / dialog" }, + { keys: ["?"], desc: "Show this help" }, + ]; + + const flyout = document.createElement("div"); + flyout.className = "shortcut-flyout"; + flyout.setAttribute("aria-hidden", "true"); + flyout.setAttribute("role", "dialog"); + flyout.setAttribute("aria-modal", "true"); + flyout.setAttribute("aria-label", "Keyboard shortcuts"); + flyout.innerHTML = '
    ' + + '
    ' + + '
    Keyboard shortcuts
    ' + + '
    ' + + SHORTCUTS.map((s) => { + return ( + "
    " + + s.keys + .map((k) => "" + k + "") + .join(" + ") + + "
    " + + "
    " + s.desc + "
    " + ); + }).join("") + + "
    " + + "
    "; + document.body.appendChild(flyout); + + const backdrop = flyout.querySelector(".shortcut-flyout-backdrop"); + const panel = flyout.querySelector(".shortcut-flyout-panel"); + let lastFocused = null; + + function open() { + lastFocused = document.activeElement; + flyout.classList.add("is-open"); + flyout.setAttribute("aria-hidden", "false"); + panel.focus(); + } + + function close() { + flyout.classList.remove("is-open"); + flyout.setAttribute("aria-hidden", "true"); + if (lastFocused && typeof lastFocused.focus === "function") { + lastFocused.focus(); + } + lastFocused = null; + } + + function toggle() { + if (flyout.classList.contains("is-open")) { + close(); + } + else { + open(); + } + } + + backdrop.addEventListener("click", close); + + document.addEventListener("keydown", (e) => { + const tag = document.activeElement && document.activeElement.tagName; + const isOpen = flyout.classList.contains("is-open"); + if (!isOpen && (tag === "INPUT" || tag === "TEXTAREA")) { + return; + } + if (e.key === "?") { + e.preventDefault(); + toggle(); + } + else if (e.key === "Escape" && isOpen) { + close(); + } + else if (e.key === "Tab" && isOpen) { + // Dialog has no focusable children: keep focus on the panel. + e.preventDefault(); + panel.focus(); + } + }); +})(); diff --git a/docs/_static/js/sidebar-mobile.js b/docs/_static/js/sidebar-mobile.js new file mode 100644 index 0000000000..dc6693f383 --- /dev/null +++ b/docs/_static/js/sidebar-mobile.js @@ -0,0 +1,168 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Mobile sidebar: hamburger button toggles a `.sidebar-open` class on +// , CSS slides the sidebar in from the left. Tapping the +// backdrop or swiping left closes it. + +(function () { + const btn = document.getElementById("sidebar-toggle"); + const sidebar = document.querySelector(".left-sidebar"); + const body = document.body; + + if (!btn || !sidebar) { + return; + } + + const backdrop = document.createElement("div"); + backdrop.className = "sidebar-backdrop"; + backdrop.setAttribute("aria-hidden", "true"); + body.appendChild(backdrop); + + function open() { + body.classList.add("sidebar-open"); + btn.setAttribute("aria-expanded", "true"); + const firstLink = sidebar.querySelector("a"); + if (firstLink) { + firstLink.focus(); + } + } + + function close() { + body.classList.remove("sidebar-open"); + btn.setAttribute("aria-expanded", "false"); + // Restore focus to the hamburger only if it's still visible + // (offsetParent is null when display:none, e.g. >1024px). + if (btn.offsetParent !== null) { + btn.focus(); + } + } + + function toggle() { + if (body.classList.contains("sidebar-open")) { + close(); + } + else { + open(); + } + } + + btn.addEventListener("click", (e) => { + e.stopPropagation(); + toggle(); + }); + + backdrop.addEventListener("click", close); + + let dragStartX = null; + let dragStartY = null; + let dragging = false; + const SWIPE_CLOSE_PX = 40; + + document.addEventListener( + "touchstart", + (e) => { + if (!body.classList.contains("sidebar-open")) { + return; + } + dragStartX = e.touches[0].clientX; + dragStartY = e.touches[0].clientY; + dragging = false; + }, + { passive: true }, + ); + + document.addEventListener( + "touchmove", + (e) => { + if (dragStartX === null) { + return; + } + const dx = e.touches[0].clientX - dragStartX; + const dy = e.touches[0].clientY - dragStartY; + if (!dragging) { + // Let vertical scrolls through; only engage on a clear + // leftward drag. + if (Math.abs(dx) <= Math.abs(dy)) { + dragStartX = null; + return; + } + if (dx > -8) { + return; + } + dragging = true; + } + // Follow the finger 1:1, no transition so it tracks live. + sidebar.style.transition = "none"; + sidebar.style.transform = "translateX(" + Math.min(0, dx) + "px)"; + }, + { passive: true }, + ); + + document.addEventListener("touchend", (e) => { + if (dragStartX === null) { + return; + } + const dx = e.changedTouches[0].clientX - dragStartX; + dragStartX = null; + if (!dragging) { + // A quick flick with no visible drag still closes (CSS + // transition animates it). + if (dx < -SWIPE_CLOSE_PX) { + close(); + } + return; + } + dragging = false; + const willClose = dx < -SWIPE_CLOSE_PX; + requestAnimationFrame(() => { + sidebar.style.transition = ""; + sidebar.style.transform = ""; + if (willClose) { + close(); + } + }); + }); + + sidebar.addEventListener("click", (e) => { + const link = e.target.closest("a"); + if (link) { + close(); + } + }); + + document.addEventListener("keydown", (e) => { + if (!body.classList.contains("sidebar-open")) { + return; + } + if (e.key === "Escape") { + close(); + return; + } + if (e.key === "Tab") { + const items = sidebar.querySelectorAll("a, button, input"); + if (!items.length) { + return; + } + const first = items[0]; + const last = items[items.length - 1]; + if (e.shiftKey && document.activeElement === first) { + e.preventDefault(); + last.focus(); + } + else if (!e.shiftKey && document.activeElement === last) { + e.preventDefault(); + first.focus(); + } + } + }); + + // Auto-close when transitioning to desktop width. + const desktop = window.matchMedia("(min-width: 1025px)"); + desktop.addEventListener("change", (e) => { + if (e.matches && body.classList.contains("sidebar-open")) { + close(); + } + }); +})(); diff --git a/docs/_static/js/theme-toggle.js b/docs/_static/js/theme-toggle.js new file mode 100644 index 0000000000..a9a07a62c0 --- /dev/null +++ b/docs/_static/js/theme-toggle.js @@ -0,0 +1,68 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Flips data-theme on between "light" and "dark" and persists +// the choice in localStorage. Initial application happens inline in +// layout.html (extrahead) so there is no flash on page load. Keep the +// "psutil-sphinx-theme" key in sync with the inline script. +// +// Shortcuts: +// click on #theme-toggle button (in the topbar) +// Shift+D anywhere outside an input/textarea +(function () { + const html = document.documentElement; + const KEY = "psutil-sphinx-theme"; + + function applyTheme(dark) { + html.setAttribute("data-theme", dark ? "dark" : "light"); + const btn = document.getElementById("theme-toggle"); + if (btn) { + const icon = btn.querySelector("i"); + if (icon) { + icon.className = dark + ? "fa-solid fa-sun" + : "fa-regular fa-moon"; + } + } + } + + function setTheme(dark) { + applyTheme(dark); + try { + localStorage.setItem(KEY, dark ? "dark" : "light"); + } + catch (e) { + // private mode / storage disabled: skip persisting + } + } + + // Sync the icon with whatever the inline script in set. + applyTheme(html.getAttribute("data-theme") === "dark"); + + const btn = document.getElementById("theme-toggle"); + if (btn) { + btn.addEventListener("click", () => { + setTheme(html.getAttribute("data-theme") !== "dark"); + }); + } + + // Shift+D: toggle dark mode (skip when typing in a form field). + document.addEventListener("keydown", (e) => { + const tag = document.activeElement && document.activeElement.tagName; + if (tag === "INPUT" || tag === "TEXTAREA") { + return; + } + if (e.shiftKey && e.key === "D") { + setTheme(html.getAttribute("data-theme") !== "dark"); + } + }); + + // Keep tabs in sync: another tab toggled, mirror it here without + // re-writing localStorage (would loop the storage event). + window.addEventListener("storage", (e) => { + if (e.key === KEY && e.newValue) { + applyTheme(e.newValue === "dark"); + } + }); +})(); diff --git a/docs/_static/js/version-selector.js b/docs/_static/js/version-selector.js new file mode 100644 index 0000000000..ab82737c6f --- /dev/null +++ b/docs/_static/js/version-selector.js @@ -0,0 +1,51 @@ +// Copyright (c) 2009 Giampaolo Rodola. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Closes the topbar version dropdown on an outside click or Esc. + +(function () { + const details = document.getElementById("version-selector"); + if (!details) { + return; + } + + document.addEventListener("click", function (event) { + if (details.open && !details.contains(event.target)) { + details.removeAttribute("open"); + } + }); + + document.addEventListener("keydown", function (event) { + if (event.key === "Escape" && details.open) { + details.removeAttribute("open"); + details.querySelector("summary").focus(); + } + }); + + // Land on the same page in the other version when it exists, else follow + // the link to that version's home page. + const page = details.dataset.versionPage; + if (!page) { + return; + } + for (const link of details.querySelectorAll(".topbar-versions-item")) { + if (link.classList.contains("is-current")) { + continue; + } + link.addEventListener("click", function (event) { + if (event.metaKey || event.ctrlKey || event.shiftKey) { + return; + } + const same = new URL(page + "/", link.href).href; + event.preventDefault(); + fetch(same, { method: "HEAD" }) + .then((resp) => { + window.location.href = resp.ok ? same : link.href; + }) + .catch(() => { + window.location.href = link.href; + }); + }); + } +})(); diff --git a/docs/_static/logo.png b/docs/_static/logo.png deleted file mode 100644 index 7d975ec9d2..0000000000 Binary files a/docs/_static/logo.png and /dev/null differ diff --git a/docs/_static/sidebar.js b/docs/_static/sidebar.js deleted file mode 100644 index 3376963911..0000000000 --- a/docs/_static/sidebar.js +++ /dev/null @@ -1,161 +0,0 @@ -/* - * sidebar.js - * ~~~~~~~~~~ - * - * This script makes the Sphinx sidebar collapsible. - * - * .sphinxsidebar contains .sphinxsidebarwrapper. This script adds in - * .sphixsidebar, after .sphinxsidebarwrapper, the #sidebarbutton used to - * collapse and expand the sidebar. - * - * When the sidebar is collapsed the .sphinxsidebarwrapper is hidden and the - * width of the sidebar and the margin-left of the document are decreased. - * When the sidebar is expanded the opposite happens. This script saves a - * per-browser/per-session cookie used to remember the position of the sidebar - * among the pages. Once the browser is closed the cookie is deleted and the - * position reset to the default (expanded). - * - * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. - * :license: BSD, see LICENSE for details. - * - */ - -$(function() { - // global elements used by the functions. - // the 'sidebarbutton' element is defined as global after its - // creation, in the add_sidebar_button function - var bodywrapper = $('.bodywrapper'); - var sidebar = $('.sphinxsidebar'); - var sidebarwrapper = $('.sphinxsidebarwrapper'); - - // original margin-left of the bodywrapper and width of the sidebar - // with the sidebar expanded - var bw_margin_expanded = bodywrapper.css('margin-left'); - var ssb_width_expanded = sidebar.width(); - - // margin-left of the bodywrapper and width of the sidebar - // with the sidebar collapsed - var bw_margin_collapsed = '.8em'; - var ssb_width_collapsed = '.8em'; - - // colors used by the current theme - var dark_color = '#AAAAAA'; - var light_color = '#CCCCCC'; - - function sidebar_is_collapsed() { - return sidebarwrapper.is(':not(:visible)'); - } - - function toggle_sidebar() { - if (sidebar_is_collapsed()) - expand_sidebar(); - else - collapse_sidebar(); - } - - function collapse_sidebar() { - sidebarwrapper.hide(); - sidebar.css('width', ssb_width_collapsed); - bodywrapper.css('margin-left', bw_margin_collapsed); - sidebarbutton.css({ - 'margin-left': '0', - //'height': bodywrapper.height(), - 'height': sidebar.height(), - 'border-radius': '5px' - }); - sidebarbutton.find('span').text('»'); - sidebarbutton.attr('title', _('Expand sidebar')); - document.cookie = 'sidebar=collapsed'; - } - - function expand_sidebar() { - bodywrapper.css('margin-left', bw_margin_expanded); - sidebar.css('width', ssb_width_expanded); - sidebarwrapper.show(); - sidebarbutton.css({ - 'margin-left': ssb_width_expanded-12, - //'height': bodywrapper.height(), - 'height': sidebar.height(), - 'border-radius': '0 5px 5px 0' - }); - sidebarbutton.find('span').text('«'); - sidebarbutton.attr('title', _('Collapse sidebar')); - //sidebarwrapper.css({'padding-top': - // Math.max(window.pageYOffset - sidebarwrapper.offset().top, 10)}); - document.cookie = 'sidebar=expanded'; - } - - function add_sidebar_button() { - sidebarwrapper.css({ - 'float': 'left', - 'margin-right': '0', - 'width': ssb_width_expanded - 28 - }); - // create the button - sidebar.append( - '
    «
    ' - ); - var sidebarbutton = $('#sidebarbutton'); - // find the height of the viewport to center the '<<' in the page - var viewport_height; - if (window.innerHeight) - viewport_height = window.innerHeight; - else - viewport_height = $(window).height(); - var sidebar_offset = sidebar.offset().top; - - var sidebar_height = sidebar.height(); - //var sidebar_height = Math.max(bodywrapper.height(), sidebar.height()); - sidebarbutton.find('span').css({ - 'display': 'block', - 'margin-top': sidebar_height/2 - 10 - //'margin-top': (viewport_height - sidebar.position().top - 20) / 2 - //'position': 'fixed', - //'top': Math.min(viewport_height/2, sidebar_height/2 + sidebar_offset) - 10 - }); - - sidebarbutton.click(toggle_sidebar); - sidebarbutton.attr('title', _('Collapse sidebar')); - sidebarbutton.css({ - 'border-radius': '0 5px 5px 0', - 'color': '#444444', - 'background-color': '#CCCCCC', - 'font-size': '1.2em', - 'cursor': 'pointer', - 'height': sidebar_height, - 'padding-top': '1px', - 'padding-left': '1px', - 'margin-left': ssb_width_expanded - 12 - }); - - sidebarbutton.hover( - function () { - $(this).css('background-color', dark_color); - }, - function () { - $(this).css('background-color', light_color); - } - ); - } - - function set_position_from_cookie() { - if (!document.cookie) - return; - var items = document.cookie.split(';'); - for(var k=0; k{{ _('Manual') }} -{{ toctree() }} -Back to Welcome diff --git a/docs/_template/indexcontent.html b/docs/_template/indexcontent.html deleted file mode 100644 index dd5e7249a0..0000000000 --- a/docs/_template/indexcontent.html +++ /dev/null @@ -1,4 +0,0 @@ -{% extends "defindex.html" %} -{% block tables %} - -{% endblock %} diff --git a/docs/_template/indexsidebar.html b/docs/_template/indexsidebar.html deleted file mode 100644 index 903675d100..0000000000 --- a/docs/_template/indexsidebar.html +++ /dev/null @@ -1,8 +0,0 @@ -

    Useful links

    - diff --git a/docs/_template/page.html b/docs/_template/page.html deleted file mode 100644 index 04b47b4153..0000000000 --- a/docs/_template/page.html +++ /dev/null @@ -1,66 +0,0 @@ -{% extends "!page.html" %} -{% block extrahead %} -{{ super() }} -{% if not embedded %}{% endif %} - - -{% endblock %} - -{% block rootrellink %} -
  • Project Homepage{{ reldelim1 }}
  • -
  • {{ shorttitle }}{{ reldelim1 }}
  • -{% endblock %} - - -{% block footer %} - -{% endblock %} \ No newline at end of file diff --git a/docs/_templates/ablog/collection.html b/docs/_templates/ablog/collection.html new file mode 100644 index 0000000000..50ae2ca155 --- /dev/null +++ b/docs/_templates/ablog/collection.html @@ -0,0 +1,71 @@ +{#- + Override of ablog/collection.html. Renders blog.html (all posts), + tag pages, year archives. Card-style listing inspired by + blog.python.org. +-#} +{%- extends "page.html" %} + +{%- macro postlink(post) -%} + {%- if post.external_link -%}{{ post.external_link }} + {%- else -%}{{ pathto(post.docname) }}{{ anchor(post) }} + {%- endif -%} +{%- endmacro -%} + +{%- set month_abbr = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] -%} + +{% block body %} +
    +
    +

    + {%- if pagename == ablog.blog_path -%} + {{ _('All posts') }} + {%- else -%} + {%- set is_tag = '/tag/' in pagename -%} + {%- set lq = '“' if is_tag else '' -%} + {%- set rq = 'â€' if is_tag else '' -%} + {{ header }} {{ lq }}{% if collection.href %}{{ collection }}{% else %}{{ collection }}{% endif %}{{ rq }} + {%- endif -%} +

    +
    + +
      + {% for post in collection %} + {% set ns = namespace(featured=false) %} + {% for tag in post.tags or [] %}{% if tag|string == 'featured' %}{% set ns.featured = true %}{% endif %}{% endfor %} + {% set featured = ns.featured %} +
    1. +
      + + {% if post.published %} + + {% else %} + {{ _('Draft') }} + {% endif %} + +
      +

      + {{ post.title }} +

      + {% if featured %}{% endif %} +
      +
      + {% if post.excerpt %} +
      {{ post.to_html(pagename) | safe }}
      + {% endif %} + {% if post.tags %} +
        + {% for tag in post.tags|sort %} + {% if tag|string != 'featured' %} +
      • {{ tag }}
      • + {% endif %} + {% endfor %} +
      + {% endif %} +
    2. + {% endfor %} +
    +
    +{% if giscus_repo %} + +{% endif %} +{% endblock body %} diff --git a/docs/_templates/ablog/postnavy.html b/docs/_templates/ablog/postnavy.html new file mode 100644 index 0000000000..dced364332 --- /dev/null +++ b/docs/_templates/ablog/postnavy.html @@ -0,0 +1,28 @@ +{#- + Override of ablog/postnavy.html: emit the same markup as the doc + prev/next at the bottom of regular pages, so a single CSS rule + (prev-next.css) styles both. +-#} +{% set post = ablog[pagename] %} +{% if post.published and ablog.post_show_prev_next and (post.prev or post.next) %} + +{% endif %} diff --git a/docs/_templates/banner.html b/docs/_templates/banner.html new file mode 100644 index 0000000000..a268cce7fe --- /dev/null +++ b/docs/_templates/banner.html @@ -0,0 +1,13 @@ +{#- + Dismissal is remembered per data-banner-id. The message below is + TEMPORARY, the component isn't. +-#} +
    + + You're reading the development version of the docs. + For the current release see psutil.readthedocs.io/stable. + + +
    diff --git a/docs/_templates/breadcrumbs.html b/docs/_templates/breadcrumbs.html new file mode 100644 index 0000000000..99d0e607b6 --- /dev/null +++ b/docs/_templates/breadcrumbs.html @@ -0,0 +1,30 @@ +{#- + Breadcrumbs / section banner. Currently used only by the blog + sub-site to render a "Blog" banner at the top of every blog page, + so the visitor knows they're in the blog area. Other pages render + nothing here. +-#} +{%- if ablog is defined %} +{%- set in_blog = pagename == ablog.blog_path or pagename.startswith(ablog.blog_path ~ '/') %} +{%- if in_blog %} +
    + + + + RSS + +
    +{%- endif %} +{%- endif %} diff --git a/docs/_templates/comments.html b/docs/_templates/comments.html new file mode 100644 index 0000000000..ee1638b30e --- /dev/null +++ b/docs/_templates/comments.html @@ -0,0 +1,21 @@ +{#- + Blog comments (giscus). Post pages only, unless the post carries a + bare ":no_comments:" field. Config comes from conf.py via + _ext/giscus.py. + + Hidden until js/giscus.js has the widget up, so a reader with JS + off gets nothing rather than an empty box. data-term is the source + path, not the URL, so a domain change keeps threads attached. +-#} +{% if is_blog_post and giscus_repo %} + +{% endif %} diff --git a/docs/_templates/footer.html b/docs/_templates/footer.html new file mode 100644 index 0000000000..2e87d8e2fe --- /dev/null +++ b/docs/_templates/footer.html @@ -0,0 +1,40 @@ + diff --git a/docs/_templates/globaltoc.html b/docs/_templates/globaltoc.html new file mode 100644 index 0000000000..cab494fc18 --- /dev/null +++ b/docs/_templates/globaltoc.html @@ -0,0 +1,5 @@ +{# Override of basic theme's globaltoc.html: render only the toctree, + without the "Table of Contents" h3 heading. #} +{{ toctree(includehidden=theme_globaltoc_includehidden, + collapse=theme_globaltoc_collapse, + maxdepth=theme_globaltoc_maxdepth) }} diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 0000000000..5f81a8aef2 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,121 @@ +{#- + psutil-sphinx-theme: page layout. + + Skeleton: topbar / left sidebar / main column (article + right + sidebar) / prev-next / footer. Each chunk lives in its own include + to keep this file scan-able. Built on Sphinx's `basic` theme, + overriding only the blocks needed to assemble the grid. +-#} +{% extends "basic/layout.html" %} + +{#- Bumping this brings the banner back for anyone who dismissed the + previous message. -#} +{% set banner_id = 'dev' %} + +{% block htmltitle %} +{%- if pagename == root_doc -%} +{{ _('psutil: Process and System Utilities for Python') }} +{%- else -%} +{{ super() }} +{%- endif -%} +{% endblock %} +{% block extrahead %} +{{ super() }} + +{%- set _parts = pagename.split('/') %} +{%- set _collections = ['blog/tag', 'blog/category', 'blog/author', + 'blog/archive', 'blog/drafts'] %} +{%- set _year = _parts | length == 2 and _parts[0] == 'blog' + and _parts[1].isdigit() %} +{%- if pagename in ['genindex', 'py-modindex', '404'] + or _parts[:2] | join('/') in _collections + or _year %} + +{%- endif %} + +{# Larger icon for iOS home-screen / share sheet. #} + + + + +{# GoatCounter. Dashboard: https://psutil.goatcounter.com #} + + +{# Google Analytics #} + + +{% endblock %} + +{# Expose the page name on so CSS can match the active sidebar + item, whose link Sphinx renders as href="#". #} +{% block body_tag %}{% endblock %} + +{% block header %} +
    + {% include "banner.html" %} + {% include "topbar.html" %} +
    +{% endblock %} + +{# Suppress basic's default top/bottom breadcrumb. #} +{% block relbar1 %}{% endblock %} +{% block relbar2 %}{% endblock %} + +{# Suppress basic's body-level footer; ours is rendered in-column + below, inside .main, by footer.html. #} +{% block footer %}{% endblock %} + +{# Pages that don't get a right-side TOC. The article column expands + to fill the row. #} +{% set _no_toc_pages = ['index', 'search', 'genindex'] %} +{% set _has_toc = pagename not in _no_toc_pages %} + +{% block content %} +
    + {% include "sidebar.html" %} +
    + {% include "breadcrumbs.html" %} +
    +
    +
    + {% block body %}{% endblock %} +
    + {% include "prev-next.html" %} + {% include "comments.html" %} +
    + + {% if _has_toc %}{% include "toc.html" %}{% endif %} +
    +
    + + {# Footer lives outside
    so its contentinfo landmark is top-level. #} + {% include "footer.html" %} +
    +{% endblock %} diff --git a/docs/_templates/prev-next.html b/docs/_templates/prev-next.html new file mode 100644 index 0000000000..507bb06b2d --- /dev/null +++ b/docs/_templates/prev-next.html @@ -0,0 +1,26 @@ +{#- + Prev / next page navigation. Renders below the article, full width + of the main column. Styled by static/css/prev-next.css. +-#} +{% if prev or next %} + +{% endif %} diff --git a/docs/_templates/search.html b/docs/_templates/search.html new file mode 100644 index 0000000000..6e9223d954 --- /dev/null +++ b/docs/_templates/search.html @@ -0,0 +1,33 @@ +{%- extends "!search.html" %} + +{# Suppress basic theme's "Searching for multiple words…" line, + redundant once we have our own prominent input at the top. #} +{% block searchtext %}{% endblock %} + +{# Replace the basic theme's tiny default form with our prominent + one. The hidden inputs preserve search semantics. #} +{% block searchbox %} +{# Empty action submits to the current URL, so this works under any + builder (dirhtml serves this page as /search/, not search.html). #} +
    + + + + + +
    + +{% endblock %} diff --git a/docs/_templates/sidebar.html b/docs/_templates/sidebar.html new file mode 100644 index 0000000000..2015439e63 --- /dev/null +++ b/docs/_templates/sidebar.html @@ -0,0 +1,17 @@ +{#- + Left sidebar: search box at top, then global table of contents + (toctree). Sticky to the viewport, scrolls independently of the + main column. Styled by static/css/left-sidebar.css. +-#} + diff --git a/docs/_templates/toc.html b/docs/_templates/toc.html new file mode 100644 index 0000000000..bded595c1e --- /dev/null +++ b/docs/_templates/toc.html @@ -0,0 +1,52 @@ +{#- + Right-side per-page TOC. Three modes: + + 1. Blog: show tags sorted by post count. + 2. Glossary page: show list of terms. + 3. Default: Sphinx's {{ toc }} headings tree. + + Renders nothing when there's no content (e.g. py-modindex, blog + landing pages with an empty toc). +-#} +{%- set _is_blog_collection = collection is defined and collection -%} +{%- if _is_blog_collection or glossary_terms or (toc and toc.strip()) -%} +{%- if glossary_terms -%} +{%- set _aside_class = "right-sidebar right-sidebar--glossary" -%} +{%- else -%} +{%- set _aside_class = "right-sidebar" -%} +{%- endif -%} + +{%- endif -%} diff --git a/docs/_templates/topbar.html b/docs/_templates/topbar.html new file mode 100644 index 0000000000..6c2301095e --- /dev/null +++ b/docs/_templates/topbar.html @@ -0,0 +1,73 @@ +{#- + Top bar: logo on the left, action cluster (Blog, theme toggle, + GitHub stars, version selector) on the right. Fixed at the top of + the viewport. Styled by static/css/topbar.css. + + The version list comes from html_context in conf.py. +-#} + diff --git a/docs/_themes/pydoctheme/static/pydoctheme.css b/docs/_themes/pydoctheme/static/pydoctheme.css deleted file mode 100644 index 4196e5582c..0000000000 --- a/docs/_themes/pydoctheme/static/pydoctheme.css +++ /dev/null @@ -1,187 +0,0 @@ -@import url("default.css"); - -body { - background-color: white; - margin-left: 1em; - margin-right: 1em; -} - -div.related { - margin-bottom: 1.2em; - padding: 0.5em 0; - border-top: 1px solid #ccc; - margin-top: 0.5em; -} - -div.related a:hover { - color: #0095C4; -} - -div.related:first-child { - border-top: 0; - padding-top: 0; - border-bottom: 1px solid #ccc; -} - -div.sphinxsidebar { - background-color: #eeeeee; - border-radius: 5px; - line-height: 130%; - font-size: smaller; -} - -div.sphinxsidebar h3, div.sphinxsidebar h4 { - margin-top: 1.5em; -} - -div.sphinxsidebarwrapper > h3:first-child { - margin-top: 0.2em; -} - -div.sphinxsidebarwrapper > ul > li > ul > li { - margin-bottom: 0.4em; -} - -div.sphinxsidebar a:hover { - color: #0095C4; -} - -div.sphinxsidebar input { - font-family: 'Lucida Grande','Lucida Sans','DejaVu Sans',Arial,sans-serif; - border: 1px solid #999999; - font-size: smaller; - border-radius: 3px; -} - -div.sphinxsidebar input[type=text] { - max-width: 150px; -} - -div.body { - padding: 0 0 0 1.2em; -} - -div.body p { - line-height: 140%; -} - -div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { - margin: 0; - border: 0; - padding: 0.3em 0; -} - -div.body hr { - border: 0; - background-color: #ccc; - height: 1px; -} - -div.body pre { - border-radius: 3px; - border: 1px solid #ac9; -} - -div.body div.admonition, div.body div.impl-detail { - border-radius: 3px; -} - -div.body div.impl-detail > p { - margin: 0; -} - -div.body div.seealso { - border: 1px solid #dddd66; -} - -div.body a { - color: #00608f; -} - -div.body a:visited { - color: #30306f; -} - -div.body a:hover { - color: #00B0E4; -} - -tt, pre { - font-family: monospace, sans-serif; - font-size: 96.5%; -} - -div.body tt { - border-radius: 3px; -} - -div.body tt.descname { - font-size: 120%; -} - -div.body tt.xref, div.body a tt { - font-weight: normal; -} - -p.deprecated { - border-radius: 3px; -} - -table.docutils { - border: 1px solid #ddd; - min-width: 20%; - border-radius: 3px; - margin-top: 10px; - margin-bottom: 10px; -} - -table.docutils td, table.docutils th { - border: 1px solid #ddd !important; - border-radius: 3px; -} - -table p, table li { - text-align: left !important; -} - -table.docutils th { - background-color: #eee; - padding: 0.3em 0.5em; -} - -table.docutils td { - background-color: white; - padding: 0.3em 0.5em; -} - -table.footnote, table.footnote td { - border: 0 !important; -} - -div.footer { - line-height: 150%; - margin-top: -2em; - text-align: right; - width: auto; - margin-right: 10px; -} - -div.footer a:hover { - color: #0095C4; -} - -div.body h1, -div.body h2, -div.body h3 { - background-color: #EAEAEA; - border-bottom: 1px solid #CCC; - padding-top: 2px; - padding-bottom: 2px; - padding-left: 5px; - margin-top: 5px; - margin-bottom: 5px; -} - -div.body h2 { - padding-left:10px; -} diff --git a/docs/_themes/pydoctheme/theme.conf b/docs/_themes/pydoctheme/theme.conf deleted file mode 100644 index 95b97e5369..0000000000 --- a/docs/_themes/pydoctheme/theme.conf +++ /dev/null @@ -1,23 +0,0 @@ -[theme] -inherit = default -stylesheet = pydoctheme.css -pygments_style = sphinx - -[options] -bodyfont = 'Lucida Grande', 'Lucida Sans', 'DejaVu Sans', Arial, sans-serif -headfont = 'Lucida Grande', 'Lucida Sans', 'DejaVu Sans', Arial, sans-serif -footerbgcolor = white -footertextcolor = #555555 -relbarbgcolor = white -relbartextcolor = #666666 -relbarlinkcolor = #444444 -sidebarbgcolor = white -sidebartextcolor = #444444 -sidebarlinkcolor = #444444 -bgcolor = white -textcolor = #222222 -linkcolor = #0090c0 -visitedlinkcolor = #00608f -headtextcolor = #1a1a1a -headbgcolor = white -headlinkcolor = #aaaaaa diff --git a/docs/about.rst b/docs/about.rst new file mode 100644 index 0000000000..aeee7cc1fe --- /dev/null +++ b/docs/about.rst @@ -0,0 +1,191 @@ +:orphan: + +.. _tips: + +About this site +=============== + +This site is built with `Sphinx `__, on top of its +built-in ``basic`` theme. From there it has been heavily customized: the +layout, navigation, search, dark mode, code blocks, mobile behavior, and many +small details were rebuilt or extended specifically for these docs. + +Keyboard shortcuts +------------------ + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Key + - Action + * - :kbd:`Shift+D` + - Toggle dark / light mode + * - :kbd:`Ctrl+K` + - Open search + * - :kbd:`↑` / :kbd:`↓` + - Navigate search results + * - :kbd:`Enter` + - Open highlighted search result + * - :kbd:`Escape` + - Close search / dialog + * - :kbd:`?` + - Show this help + +Dark mode +--------- + +.. |moon| raw:: html + + + +By default the site matches your operating system's light or dark setting. +Click the |moon| icon in the top-right corner, or press :kbd:`Shift+D` +anywhere, to override it. Your choice is saved and restored on your next visit. + +.. container:: about-shot-pair + + .. image:: /_static/images/about/darkmode-light.png + :alt: Light mode + :class: about-shot + + .. image:: /_static/images/about/darkmode-dark.png + :alt: Dark mode + :class: about-shot + +Documentation versions +---------------------- + +.. |tag| raw:: html + + + +The |tag| menu in the top-right corner switches between this development +version of the docs, built from the master branch, and past releases. + +.. image:: /_static/images/about/versions.png + :alt: Version selector + :class: about-shot + +Clickable API references +------------------------ + +Identifiers in code blocks (e.g. ``p = psutil.Process()``, ``p.cpu_percent()``) +are clickable. Hover over one of those to see the highlight, then click to jump +to the corresponding API reference doc. This is powered by +`sphinx-codeautolink `__. + +.. image:: /_static/images/about/clickable.png + :alt: Clickable API reference + :class: about-shot + +Copy buttons +------------ + +Every code block has a copy button in its top-right corner which appears on +hover. For interactive (``>>>``) examples it copies just the code, leaving out +the prompts and the output, so you can paste it straight into a shell. + +.. image:: /_static/images/about/copy.png + :alt: Copy button + :class: about-shot + +Copy page +--------- + +Next to the title of every documentation page there's a *Copy page* button, +which copies the page's RsT source to your clipboard. Handy for for pasting a +whole page into an LLM. + +.. image:: /_static/images/about/copypage.png + :alt: Copy page button + :class: about-shot + +Search +------ + +Press :kbd:`Ctrl+K` to jump straight to the search box. Once results appear, +use the :kbd:`↑` and :kbd:`↓` arrow keys to move through them and :kbd:`Enter` +to open one. + +.. image:: /_static/images/about/search.png + :alt: Search results + :class: about-shot + +TOC / On this page +------------------ + +Wide screens show an "On this page" sidebar on the right, listing the current +page's sections. As you scroll it highlights the section you're in, and +clicking an entry jumps straight to it. + +.. image:: /_static/images/about/toc.png + :alt: On this page outline + :class: about-shot + +Back to top +----------- + +On a long page, start scrolling back up and a small button will appear in the +bottom-right corner. Click it to go back to the top of the page. + +.. image:: /_static/images/about/backtotop.png + :alt: Back-to-top button + :class: about-shot + +Reading on mobile +----------------- + +On a phone the navigation collapses behind the **☰** button in the top bar. Tap +it to open the menu, then swipe left anywhere on the page (or tap outside it) +to close it again. + +.. image:: /_static/images/about/mobile.png + :alt: Mobile sidebar + :class: about-shot + +Blog +---- + +New releases and deep-dives are written up on the :doc:`blog `. Blog +provides a RSS feed (the icon at the top of the blog page) so you can receive +notifications of new blog posts from your reader + a commenting system provided +by `giscus `_. + +Other internal optimizations +---------------------------- + +- Fonts and icons are served directly from this site, not from third-party CDNs + such as Google Fonts. This means the docs renders the same from all countries + (e.g. mainland China). Each font includes only the glyphs used by the site, + keeping page weight small. + +- Links to this site shared on social medias show rich preview cards. + +- Every page declares a canonical URL and is listed in a generated + ``sitemap.xml``, so search engines can discover and index the whole site. + +- The docs build is strict: Python code snippets are syntax-checked, and broken + links or unresolved API cross-references make the build fail. + +Linking from your own docs +-------------------------- + +Other Sphinx projects can cross-reference psutil's API directly through the +`intersphinx `_ +extension. Add psutil to ``intersphinx_mapping`` in your ``conf.py``: + +.. code-block:: python + + intersphinx_mapping = { + "psutil": ("https://psutil.io/", None), + } + +You can then reference any psutil object and it links straight here, e.g.: + +.. code-block:: rst + + See :func:`psutil.cpu_times` function. + +This will automatically turn ``psutil.cpu_times`` into a link pointing to this +site. diff --git a/docs/adoption.rst b/docs/adoption.rst new file mode 100644 index 0000000000..4360d563f3 --- /dev/null +++ b/docs/adoption.rst @@ -0,0 +1,213 @@ +Who uses psutil +=============== + +.. Numbers below are refreshed by `make refresh-adoption-stats`, + which writes them in-place from PyPI / GitHub. Run it before + tagging a release. + +psutil is among the +`top 100 `__ most-downloaded +packages on PyPI, with **390+ million** downloads per month and **780,000+** +`GitHub repositories `__ +using it. The projects below are a small sample of notable software that +depends on psutil. See also :doc:`alternatives` for related Python libraries +and equivalents in other languages. + +Infrastructure / automation +--------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 28 32 14 26 + + * - Project + - Description + - Stars + - psutil usage + * - `Home Assistant `__ + - Open source home automation platform + - |homeassistant-stars| + - system monitor integration + * - `Ansible `__ + - IT automation platform + - |ansible-stars| + - system fact gathering + * - `Apache Airflow `__ + - Workflow orchestration platform + - |airflow-stars| + - process supervisor, unit testing + * - `Celery `__ + - Distributed task queue + - |celery-stars| + - worker process monitoring, memleak detection + * - `Salt `__ + - Infrastructure automation at scale + - |salt-stars| + - deep system data collection (grains) + * - `Dask `__ + - Parallel computing with task scheduling + - |dask-stars| + - `metrics dashboard `__, profiling + * - `Ajenti `__ + - Web-based server administration panel + - |ajenti-stars| + - monitoring plugins, deep integration + +AI / machine learning +--------------------- + +.. list-table:: + :header-rows: 1 + :widths: 28 32 14 26 + + * - Project + - Description + - Stars + - psutil usage + * - `TensorFlow `__ + - Open source machine learning framework by Google + - |tensorflow-stars| + - unit tests + * - `PyTorch `__ + - Tensors and dynamic neural networks with GPU acceleration + - |pytorch-stars| + - benchmark scripts + * - `Ray `__ + - AI compute engine with distributed runtime + - |ray-stars| + - metrics dashboard + * - `MLflow `__ + - AI/ML engineering platform + - |mlflow-stars| + - deep system monitoring integration + +Developer tools +--------------- + +.. list-table:: + :header-rows: 1 + :widths: 28 32 14 26 + + * - Project + - Description + - Stars + - psutil usage + * - `Sentry `__ + - Error tracking and performance monitoring + - |sentry-stars| + - send telemetry metrics + * - `Locust `__ + - Scalable load testing in Python + - |locust-stars| + - monitoring of the Locust process + * - `Spyder `__ + - Scientific Python IDE + - |spyder-stars| + - deep integration, UI stats, process management + * - `psleak `__ + - Test framework to detect memory leaks in Python C extensions + - |psleak-stars| + - heap process memory (:func:`heap_info()`) + +System monitoring +----------------- + +.. list-table:: + :header-rows: 1 + :widths: 28 32 14 26 + + * - Project + - Description + - Stars + - psutil usage + * - `Glances `__ + - System monitoring tool (top/htop alternative) + - |glances-stars| + - core dependency for all metrics + * - `bpytop `__ + - Terminal resource monitor + - |bpytop-stars| + - core dependency for all metrics + * - `auto-cpufreq `__ + - Automatic CPU speed and power optimizer for Linux + - |auto-cpufreq-stars| + - core dependency for CPU monitoring + * - `GRR `__ + - Remote live forensics by Google + - |grr-stars| + - core dependency for system data collection + * - `s-tui `__ + - Terminal CPU stress and monitoring utility + - |stui-stars| + - core dependency for metrics + * - `asitop `__ + - Apple Silicon performance monitoring CLI + - |asitop-stars| + - core dependency for system metrics + * - `psdash `__ + - Web dashboard using psutil and Flask + - |psdash-stars| + - core dependency for all metrics + * - `dd-agent `__ + - Original monitoring agent by Datadog + - |dd-agent-stars| + - system metrics collection + * - `dd-trace-py `__ + - Python tracing and profiling library + - |ddtrace-stars| + - system metrics collection + +How this list was compiled +-------------------------- + +- `GitHub dependency graph `__ + was used to identify packages and repositories that depend on psutil. +- GitHub code search with query "psutil in:readme language:Python", sorted by + stars, was used to find additional projects that mention psutil in their + README. +- Each candidate was then manually verified by checking the project's + pyproject.toml, setup.py, setup.cfg or requirements.txt to confirm that + psutil is an actual dependency (direct, build-time, or optional), not just a + passing mention. +- Projects were excluded if they only mention psutil in documentation or + examples without declaring it as a dependency. +- Star counts are pulled dynamically from `shields.io `__ + badges. +- The final list was manually curated to include notable projects and + meaningful usages of psutil across different areas of the Python ecosystem. + +.. ============================================================================ +.. ============================================================================ +.. ============================================================================ + +.. Star badges +.. ============================================================================ + +.. |airflow-stars| image:: https://img.shields.io/github/stars/apache/airflow.svg?style=social&label=%20 +.. |ajenti-stars| image:: https://img.shields.io/github/stars/ajenti/ajenti.svg?style=social&label=%20 +.. |ansible-stars| image:: https://img.shields.io/github/stars/ansible/ansible.svg?style=social&label=%20 +.. |asitop-stars| image:: https://img.shields.io/github/stars/tlkh/asitop.svg?style=social&label=%20 +.. |auto-cpufreq-stars| image:: https://img.shields.io/github/stars/AdnanHodzic/auto-cpufreq.svg?style=social&label=%20 +.. |bpytop-stars| image:: https://img.shields.io/github/stars/aristocratos/bpytop.svg?style=social&label=%20 +.. |celery-stars| image:: https://img.shields.io/github/stars/celery/celery.svg?style=social&label=%20 +.. |dask-stars| image:: https://img.shields.io/github/stars/dask/dask.svg?style=social&label=%20 +.. |ddtrace-stars| image:: https://img.shields.io/github/stars/DataDog/dd-trace-py.svg?style=social&label=%20 +.. |dd-agent-stars| image:: https://img.shields.io/github/stars/DataDog/dd-agent.svg?style=social&label=%20 +.. |glances-stars| image:: https://img.shields.io/github/stars/nicolargo/glances.svg?style=social&label=%20 +.. |grr-stars| image:: https://img.shields.io/github/stars/google/grr.svg?style=social&label=%20 +.. |homeassistant-stars| image:: https://img.shields.io/github/stars/home-assistant/core.svg?style=social&label=%20 +.. |locust-stars| image:: https://img.shields.io/github/stars/locustio/locust.svg?style=social&label=%20 +.. |mlflow-stars| image:: https://img.shields.io/github/stars/mlflow/mlflow.svg?style=social&label=%20 +.. |psdash-stars| image:: https://img.shields.io/github/stars/Jahaja/psdash.svg?style=social&label=%20 +.. |psleak-stars| image:: https://img.shields.io/github/stars/giampaolo/psleak.svg?style=social&label=%20 +.. |pytorch-stars| image:: https://img.shields.io/github/stars/pytorch/pytorch.svg?style=social&label=%20 +.. |ray-stars| image:: https://img.shields.io/github/stars/ray-project/ray.svg?style=social&label=%20 +.. |salt-stars| image:: https://img.shields.io/github/stars/saltstack/salt.svg?style=social&label=%20 +.. |sentry-stars| image:: https://img.shields.io/github/stars/getsentry/sentry.svg?style=social&label=%20 +.. |spyder-stars| image:: https://img.shields.io/github/stars/spyder-ide/spyder.svg?style=social&label=%20 +.. |stui-stars| image:: https://img.shields.io/github/stars/amanusk/s-tui.svg?style=social&label=%20 +.. |tensorflow-stars| image:: https://img.shields.io/github/stars/tensorflow/tensorflow.svg?style=social&label=%20 + +.. --- Notes +.. Stars shield: +.. https://shields.io/badges/git-hub-repo-stars diff --git a/docs/alternatives.rst b/docs/alternatives.rst new file mode 100644 index 0000000000..9dee2ba145 --- /dev/null +++ b/docs/alternatives.rst @@ -0,0 +1,183 @@ +Alternatives +============ + +This page describes Python tools and modules that overlap with psutil, to help +you pick the right tool for the job. See also :doc:`adoption` for notable +projects that use psutil. + +Python standard library +----------------------- + +.. seealso:: + :doc:`stdlib-equivalents` for a detailed function-by-function comparison. + +os module +^^^^^^^^^ + +The :mod:`os` module provides a handful of process-related functions: +:func:`os.getpid`, :func:`os.getppid`, :func:`os.getuid`, :func:`os.cpu_count`, +:func:`os.getloadavg` (UNIX only). These are cheap wrappers around POSIX +syscalls and are perfectly fine when you only need information about the +*current* process and don't need cross-platform code. + +psutil goes further in several directions. Its primary goal is to provide a +**single portable interface** for concepts that are traditionally UNIX-only. +Things like process CPU and memory usage, open file descriptors, network +connections, signals, :term:`nice` levels, and I/O counters exist as +first-class OS primitives on Linux and macOS, but have no direct equivalent on +Windows. psutil implements all of them on Windows too (using Win32 APIs, +``NtQuerySystemInformation`` and WMI) so that code written against psutil runs +unmodified on every supported platform. Beyond portability, it also exposes the +same information for *any* process (not just the current one), and returns +structured named tuples instead of raw values. + +resource module +^^^^^^^^^^^^^^^ + +:mod:`resource` (UNIX only) lets you read and set resource limits +(``RLIMIT_*``) and get basic usage counters (user/system time, page faults, I/O +ops) for the *current* process or its children via :func:`resource.getrusage`. +It is the right tool when you specifically want to enforce or inspect +``ulimit``-style limits. + +psutil's :meth:`Process.rlimit` exposes the same interface but extends it to +all processes, not just the caller. + +subprocess module +^^^^^^^^^^^^^^^^^ + +Calling tools like ``ps``, ``top``, ``netstat``, ``vmstat`` via +:mod:`subprocess` and parsing their output is fragile: output formats differ +across OS versions and locales, parsing is error-prone, and spawning a +subprocess per sample is slow. psutil reads the same kernel data sources +directly without spawning any external processes. + +platform module +^^^^^^^^^^^^^^^ + +:mod:`platform` provides information about the OS and Python runtime, such as +OS name, kernel version, architecture, and machine type. It is useful for +identifying the environment, but does not expose runtime metrics or process +information like psutil. Overlaps with psutil's OS constants (:data:`LINUX`, +:data:`WINDOWS`, :data:`MACOS`, etc.). + +/proc filesystem +^^^^^^^^^^^^^^^^ + +On Linux, ``/proc`` exposes process and system information as virtual files. +Reading :proc:`/proc/pid/status` or :proc:`/proc/meminfo` directly is fast and +has no dependencies, which is why some minimal containers or scripts do this. +The downsides are that it is Linux-only, the format may vary across kernel +versions, and you have to parse raw text yourself. psutil parses ``/proc`` +internally, exposes the same information through a consistent cross-platform +API and handles edge cases (invalid format, compatibility with old kernels, +graceful fallbacks, etc.). + +Third-party libraries +--------------------- + +Libraries that cover areas psutil does not, or that go deeper on a specific +platform or subsystem. + +.. list-table:: + :header-rows: 1 + :widths: 5 25 + :class: longtable + + * - Library + - Focus + + * - `distro `_ + - Linux distro info (name, version, codename). psutil does not + expose OS details. + + * - `GPUtil `_ / + `pynvml `_ + - NVIDIA GPU utilization and VRAM usage. + + * - `ifaddr `_ + - Network interface address enumeration. + Overlaps with :func:`net_if_addrs`. + + * - `libvirt-python `_ + - Manage KVM/QEMU/Xen VMs: enumerate guests, query + CPU/memory allocation. Complements psutil's host-level view. + + * - `prometheus_client `_ + - Export metrics to Prometheus. Use *alongside* psutil. + + * - `py-cpuinfo `_ + - CPU brand string, micro architecture, feature flags. + + * - `pyroute2 `_ + - Linux netlink (interfaces, routes, connections). + Overlaps with :func:`net_if_addrs`, :func:`net_if_stats`, + :func:`net_connections`. + + * - `pywifi `_ + - WiFi scanning, signal strength, SSID. Exposes wireless + details that :func:`net_if_addrs` does not. + + * - `pySMART `_ + - S.M.A.R.T. disk health data. Complements + :func:`disk_io_counters`. + + * - `pywin32 `_ + - Win32 API bindings (Windows only). + + * - `setproctitle `_ + - Set process title shown by ``ps``/``top``. Writes what + :meth:`Process.name` reads. + + * - `wmi `_ + - WMI interface (Windows only). + +Other languages +--------------- + +Equivalent libraries in other languages providing cross-platform system and +process information. + +.. list-table:: + :header-rows: 1 + :widths: 5 5 20 + :class: longtable + + * - Library + - Language + - Focus + + * - `gopsutil `_ + - Go + - CPU, memory, disk, network, processes. Directly inspired + by psutil and follows a similar API. + + * - `heim `_ + - Rust + - Async-first library covering CPU, memory, disk, network, + processes and sensors. + + * - `Hardware.Info `_ + - C# / .NET + - CPU, RAM, GPU, disk, network, battery. + + * - `hwinfo `_ + - C++ + - CPU, RAM, GPU, disks, mainboard. More hardware-focused. + + * - `OSHI `_ + - Java + - OS and hardware information: CPU, memory, disk, network, + processes, sensors, USB devices. + + * - `rust-psutil `_ + - Rust + - Directly inspired by psutil and follows a similar API. + + * - `sysinfo `_ + - Rust + - CPU, memory, disk, network, processes, components. + + * - `systeminformation `_ + - Node.js + - CPU, memory, disk, network, processes, battery, Docker. diff --git a/docs/api-overview.rst b/docs/api-overview.rst new file mode 100644 index 0000000000..f4096ee6f5 --- /dev/null +++ b/docs/api-overview.rst @@ -0,0 +1,573 @@ +API overview +============ + +Overview of the entire psutil API (on Linux). This serves as a quick reference +to all available functions. For detailed documentation of each function see the +full :doc:`API reference `. + +System related functions +------------------------ + +.. _api-overview-cpu: + +CPU +^^^ + +.. code-block:: pycon + + >>> import psutil + >>> + >>> psutil.cpu_times() + scputimes(user=3961.46, + nice=169.729, + system=2150.659, + idle=16900.540, + iowait=629.59, + irq=0.0, + softirq=19.42, + steal=0.0, + guest=0, + guest_nice=0.0) + >>> + >>> for x in range(3): + ... psutil.cpu_percent(interval=1) + ... + 4.0 + 5.9 + 3.8 + >>> + >>> for x in range(3): + ... psutil.cpu_percent(interval=1, percpu=True) + ... + [4.0, 6.9, 3.7, 9.2] + [7.0, 8.5, 2.4, 2.1] + [1.2, 9.0, 9.9, 7.2] + >>> + >>> for x in range(3): + ... psutil.cpu_times_percent(interval=1, percpu=False) + ... + scputimes(user=1.5, nice=0.0, system=0.5, idle=96.5, iowait=1.5, irq=0.0, softirq=0.0, steal=0.0, guest=0.0, guest_nice=0.0) + scputimes(user=1.0, nice=0.0, system=0.0, idle=99.0, iowait=0.0, irq=0.0, softirq=0.0, steal=0.0, guest=0.0, guest_nice=0.0) + scputimes(user=2.0, nice=0.0, system=0.0, idle=98.0, iowait=0.0, irq=0.0, softirq=0.0, steal=0.0, guest=0.0, guest_nice=0.0) + >>> + >>> psutil.cpu_count() + 4 + >>> psutil.cpu_count(logical=False) + 2 + >>> + >>> psutil.cpu_stats() + scpustats(ctx_switches=20455687, interrupts=6598984, soft_interrupts=2134212, syscalls=0) + >>> + >>> psutil.cpu_freq() + scpufreq(current=931.42925, min=800.0, max=3500.0) + >>> + >>> psutil.getloadavg() # also on Windows (emulated) + (3.14, 3.89, 4.67) + >>> + +.. _api-overview-memory: + +Memory +^^^^^^ + +.. code-block:: pycon + + >>> import psutil + >>> + >>> psutil.virtual_memory() + svmem(total=10367352832, + available=6472179712, + percent=37.6, + used=8186245120, + free=2181107712, + active=4748992512, + inactive=2758115328, + buffers=790724608, + cached=3500347392, + shared=787554304) + >>> + >>> psutil.swap_memory() + sswap(total=2097147904, + used=296128512, + free=1801019392, + percent=14.1, + sin=304193536, + sout=677842944) + >>> + +.. _api-overview-disks: + +Disks +^^^^^ + +.. code-block:: pycon + + >>> import psutil + >>> + >>> psutil.disk_partitions() + [sdiskpart(device='/dev/sda1', mountpoint='/', fstype='ext4', opts='rw,nosuid'), + sdiskpart(device='/dev/sda2', mountpoint='/home', fstype='ext', opts='rw')] + >>> + >>> psutil.disk_usage('/') + sdiskusage(total=21378641920, used=4809781248, free=15482871808, percent=22.5) + >>> + >>> psutil.disk_io_counters(perdisk=False) + sdiskio(read_count=719566, + write_count=1082197, + read_bytes=18626220032, + write_bytes=24081764352, + read_time=5023392, + write_time=63199568, + read_merged_count=619166, + write_merged_count=812396, + busy_time=4523412) + >>> + +.. _api-overview-network: + +Network +^^^^^^^ + +.. code-block:: pycon + + >>> import psutil + >>> + >>> psutil.net_io_counters(pernic=True) + {'eth0': netio(bytes_sent=485291293, + bytes_recv=6004858642, + packets_sent=3251564, + packets_recv=4787798, + errin=0, + errout=0, + dropin=0, + dropout=0), + 'lo': netio(bytes_sent=2838627, + bytes_recv=2838627, + packets_sent=30567, + packets_recv=30567, + errin=0, + errout=0, + dropin=0, + dropout=0)} + >>> + >>> psutil.net_connections(kind='tcp') + [sconn(fd=115, + family=, + type=, + laddr=addr(ip='10.0.0.1', port=48776), + raddr=addr(ip='93.186.135.91', port=80), + status='ESTABLISHED', + pid=1254), + sconn(fd=117, + family=, + type=, + laddr=addr(ip='10.0.0.1', port=43761), + raddr=addr(ip='72.14.234.100', port=80), + status='CLOSING', + pid=2987), + ...] + >>> + >>> psutil.net_if_addrs() + {'lo': [snicaddr(family=, + address='127.0.0.1', + netmask='255.0.0.0', + broadcast='127.0.0.1', + ptp=None), + snicaddr(family=, + address='::1', + netmask='ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', + broadcast=None, + ptp=None), + snicaddr(family=, + address='00:00:00:00:00:00', + netmask=None, + broadcast='00:00:00:00:00:00', + ptp=None)], + 'wlan0': [snicaddr(family=, + address='192.168.1.3', + netmask='255.255.255.0', + broadcast='192.168.1.255', + ptp=None), + snicaddr(family=, + address='fe80::c685:8ff:fe45:641%wlan0', + netmask='ffff:ffff:ffff:ffff::', + broadcast=None, + ptp=None), + snicaddr(family=, + address='c4:85:08:45:06:41', + netmask=None, + broadcast='ff:ff:ff:ff:ff:ff', + ptp=None)]} + >>> + >>> psutil.net_if_stats() + {'lo': snicstats(isup=True, + duplex=, + speed=0, + mtu=65536, + flags='up,loopback,running'), + 'wlan0': snicstats(isup=True, + duplex=, + speed=100, + mtu=1500, + flags='up,broadcast,running,multicast')} + >>> + +.. _api-overview-sensors: + +Sensors +^^^^^^^ + +.. code-block:: pycon + + >>> import psutil + >>> + >>> psutil.sensors_temperatures() + {'acpitz': [shwtemp(label='', current=47.0, high=103.0, critical=103.0)], + 'asus': [shwtemp(label='', current=47.0, high=None, critical=None)], + 'coretemp': [shwtemp(label='Physical id 0', current=52.0, high=100.0, critical=100.0), + shwtemp(label='Core 0', current=45.0, high=100.0, critical=100.0)]} + >>> + >>> psutil.sensors_fans() + {'asus': [sfan(label='cpu_fan', current=3200)]} + >>> + >>> psutil.sensors_battery() + sbattery(percent=93, secsleft=16628, power_plugged=False) + >>> + +Other system info +^^^^^^^^^^^^^^^^^ + +.. code-block:: pycon + + >>> import psutil + >>> + >>> psutil.users() + [suser(name='giampaolo', terminal='pts/2', host='localhost', started=1340737536.0, pid=1352), + suser(name='giampaolo', terminal='pts/3', host='localhost', started=1340737792.0, pid=1788)] + >>> + >>> psutil.boot_time() + 1365519115.0 + >>> + +.. _api-overview-processes: + +Processes +--------- + +Oneshot +^^^^^^^ + +.. code-block:: pycon + + >>> import psutil + >>> + >>> p = psutil.Process(7055) + >>> with p.oneshot(): + ... p.name() + ... p.cpu_times() + ... p.memory_info() + ... + 'python3' + pcputimes(user=1.02, system=0.31, children_user=0.32, children_system=0.1, iowait=0.0) + pmem(rss=3164160, vms=4410163, shared=897433, text=302694, data=2422374) + >>> + +Identity +^^^^^^^^ + +.. code-block:: pycon + + >>> import psutil + >>> + >>> p = psutil.Process(7055) + >>> p + psutil.Process(pid=7055, name='python3', status=, started='09:04:44') + >>> p.pid + 7055 + >>> + >>> p.name() + 'python3' + >>> + >>> p.exe() + '/usr/bin/python3' + >>> + >>> p.cwd() + '/home/giampaolo' + >>> + >>> p.cmdline() + ['/usr/bin/python3', 'main.py'] + >>> + >>> p.status() + + >>> + >>> p.create_time() + 1267551141.5019531 + >>> + >>> p.terminal() + '/dev/pts/0' + >>> + >>> p.environ() + {'GREP_OPTIONS': '--color=auto', + 'LC_PAPER': 'it_IT.UTF-8', + 'SHELL': '/bin/bash', + 'XDG_CONFIG_DIRS': '/etc/xdg/xdg-ubuntu:/usr/share/upstart/xdg:/etc/xdg', + ...} + >>> + >>> p.is_running() + True + >>> + >>> p.as_dict() + {'num_ctx_switches': pctxsw(voluntary=63, involuntary=1), + 'pid': 5457, + 'status': , + ...} + >>> + +Process tree +^^^^^^^^^^^^ + +.. code-block:: pycon + + >>> p.ppid() + 7054 + >>> p.parent() + psutil.Process(pid=4699, name='bash', status=, started='09:06:44') + >>> + >>> p.parents() + [psutil.Process(pid=4699, name='bash', started='09:06:44'), + psutil.Process(pid=4689, name='gnome-terminal-server', status=, started='0:06:44'), + psutil.Process(pid=1, name='systemd', status=, started='05:56:55')] + >>> + >>> p.children(recursive=True) + [psutil.Process(pid=29835, name='python3', status=, started='11:45:38'), + psutil.Process(pid=29836, name='python3', status=, started='11:43:39')] + >>> + +Credentials +^^^^^^^^^^^^ + +.. code-block:: pycon + + >>> p.username() + 'giampaolo' + >>> p.uids() + puids(real=1000, effective=1000, saved=1000) + >>> p.gids() + pgids(real=1000, effective=1000, saved=1000) + >>> + +CPU / scheduling +^^^^^^^^^^^^^^^^ + +.. code-block:: pycon + + >>> p.cpu_times() + pcputimes(user=1.02, system=0.31, children_user=0.32, children_system=0.1, iowait=0.0) + >>> + >>> p.cpu_percent(interval=1.0) + 12.1 + >>> + >>> p.cpu_affinity() + [0, 1, 2, 3] + >>> p.cpu_affinity([0, 1]) # set + >>> + >>> p.cpu_num() + 1 + >>> + >>> p.num_ctx_switches() + pctxsw(voluntary=78, involuntary=19) + >>> + >>> p.nice() + 0 + >>> p.nice(10) # set + >>> + >>> p.ionice(psutil.IOPRIO_CLASS_IDLE) # set IO priority + >>> p.ionice() + pionice(ioclass=, value=0) + >>> + >>> p.rlimit(psutil.RLIMIT_NOFILE, (5, 5)) # set resource limits + >>> p.rlimit(psutil.RLIMIT_NOFILE) + (5, 5) + >>> + +Memory +^^^^^^ + +.. code-block:: pycon + + >>> p.memory_info() + pmem(rss=3164160, vms=4410163, shared=897433, text=302694, data=2422374) + >>> + >>> p.memory_extras() + pmem_extras(peak_rss=4172190, + peak_vms=6399001, + rss_anon=2266726, + rss_file=897433, + rss_shmem=0, + swap_anon=0, + hugetlb=0) + >>> + >>> p.memory_percent() + 0.7823 + >>> + >>> p.memory_footprint() # "real" USS memory usage + pfootprint(uss=2355200, pss=2483712, swap=0) + >>> + >>> p.memory_maps() + pmmap_grouped(path='/lib/x8664-linux-gnu/libc-2.15.so', + rss=3821568, + size=3842048, + pss=3821568, + shared_clean=0, + shared_dirty=0, + private_clean=0, + private_dirty=3821568, + referenced=3575808, + anonymous=3821568, + swap=0), + pmmap_grouped(path='[heap]', + rss=32768, + size=139264, + pss=32768, + shared_clean=0, + shared_dirty=0, + private_clean=0, + private_dirty=32768, + referenced=32768, + anonymous=32768, + swap=0), + ...] + >>> + >>> p.page_faults() + ppagefaults(minor=5905, major=3) + >>> + +Threads +^^^^^^^ + +.. code-block:: pycon + + >>> p.threads() + [pthread(id=5234, user_time=22.5, system_time=9.2891), + pthread(id=5237, user_time=0.0707, system_time=1.1)] + >>> p.num_threads() + 4 + >>> + +Files and connections +^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: pycon + + >>> p.io_counters() + pio(read_count=478001, + write_count=59371, + read_bytes=700416, + write_bytes=69632, + read_chars=456232, + write_chars=517543) + >>> + >>> p.open_files() + [popenfile(path='/home/giampaolo/monit.py', fd=3, position=0, mode='r', flags=32768), + popenfile(path='/var/log/monit.log', fd=4, position=235542, mode='a', flags=33793)] + >>> + >>> p.net_connections(kind='tcp') + [pconn(fd=115, + family=, + type=, + laddr=addr(ip='10.0.0.1', port=48776), + raddr=addr(ip='93.186.135.91', port=80), + status=), + pconn(fd=117, + family=, + type=, + laddr=addr(ip='10.0.0.1', port=43761), + raddr=addr(ip='72.14.234.100', port=80), + status=)] + >>> + >>> p.num_fds() + 8 + >>> + +Signals +^^^^^^^ + +.. code-block:: pycon + + >>> p.send_signal(signal.SIGTERM) + >>> p.suspend() + >>> p.resume() + >>> p.terminate() + >>> p.kill() + >>> p.wait(timeout=3) + + >>> + +Other process functions +^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: pycon + + >>> import psutil + >>> + >>> psutil.pids() + [1, 2, 3, 4, 5, 6, 7, 46, 48, 50, 51, 178, 182, ...] + >>> + >>> psutil.pid_exists(3) + True + >>> + >>> for p in psutil.process_iter(['pid', 'name']): + ... print(p.pid, p.name()) + ... + 1 systemd + 2 kthreadd + 3 ksoftirqd/0 + ... + >>> + >>> def on_terminate(proc): + ... print("process {} terminated".format(proc)) + ... + >>> # waits for multiple processes to terminate + >>> gone, alive = psutil.wait_procs(procs_list, timeout=3, callback=on_terminate) + >>> + +C heap introspection +-------------------- + +.. code-block:: pycon + + >>> import psutil + >>> + >>> psutil.heap_info() + pheap(heap_used=5177792, mmap_used=819200) + >>> + >>> psutil.heap_trim() + >>> + +See also `psleak `_. + +Windows services +---------------- + +.. code-block:: pycon + + >>> import psutil + >>> + >>> list(psutil.win_service_iter()) + [, + , + , + , + ...] + >>> + >>> s = psutil.win_service_get('alg') + >>> s.as_dict() + {'binpath': 'C:\\Windows\\System32\\alg.exe', + 'description': 'Provides support for 3rd party protocol plug-ins for Internet Connection Sharing', + 'display_name': 'Application Layer Gateway Service', + 'name': 'alg', + 'pid': None, + 'start_type': 'manual', + 'status': 'stopped', + 'username': 'NT AUTHORITY\\LocalService'} + >>> diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000000..78bc4f8c08 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,2764 @@ +.. note:: + psutil 8.0 introduces breaking API changes. See the + :ref:`migration guide ` if upgrading from 7.x. + +API reference +============= + +Complete reference for all psutil classes and functions. Provided as a single +HTML page for ease of searchability. + +.. contents:: + :local: + :depth: 1 + +For a high-level overview with short examples see :doc:`api-overview`. + +System related functions +------------------------ + +CPU +^^^ + +.. function:: cpu_times(percpu=False) + + Return system CPU times as a named tuple. All fields are + :term:`cumulative counters ` (seconds) representing time + the CPU has spent in each mode since boot. The attributes availability varies + depending on the platform. Cross-platform fields: + + - :field:`user`: time spent by processes executing in user mode; on Linux + this also includes :field:`guest` time. + + - :field:`system`: time spent by processes executing in kernel mode. + + - :field:`idle`: time spent doing nothing. + + Platform-specific fields: + + - :field:`nice` *(Linux, macOS, BSD)*: time spent by :term:`niced ` + (lower-priority) processes executing in user mode; on Linux this also + includes :field:`guest_nice` time. + + - :field:`iowait` *(Linux, SunOS, AIX)*: time spent waiting for I/O to + complete (:term:`iowait`). This is *not* accounted in :field:`idle` time + counter. + + - :field:`irq` *(Linux, Windows, BSD)*: time spent for servicing + :term:`hardware interrupts `. + + - :field:`softirq` *(Linux)*: time spent for servicing + :term:`soft interrupts `. + + - :field:`steal` *(Linux)*: CPU time the virtual machine wanted to run, but + was used by other virtual machines or the host. + + - :field:`guest` *(Linux)*: time the host CPU spent running a guest operating + system (virtual machine). Already included in :field:`user` time. + + - :field:`guest_nice` *(Linux)*: like :field:`guest`, but for virtual CPUs + running at a lower :term:`nice` priority. Already included in :field:`nice` + time. + + - :field:`dpc` *(Windows)*: time spent servicing deferred procedure calls + (DPCs); DPCs are interrupts that run at a lower priority than standard + interrupts. + + When *percpu* is ``True`` return a list for each :term:`logical CPU` on the + system. The list is ordered by CPU index. The order of the list is consistent + across calls. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.cpu_times() + scputimes(user=17411.7, system=3797.02, idle=51266.57, nice=77.99, iowait=732.58, irq=0.01, softirq=142.43, steal=0.0, guest=0.0, guest_nice=0.0) + + .. note:: + CPU times are always supposed to increase over time, or at least remain the + same, and that's because time cannot go backwards. Surprisingly sometimes + this might not be the case (at least on Windows and Linux), see + `#1210 `_. + + .. versionchanged:: 8.0.0 + Windows: :field:`interrupt` field was renamed to :field:`irq`; + :field:`interrupt` still works but raises :exc:`DeprecationWarning`. + + .. versionchanged:: 8.0.0 + field order was standardized: :field:`user`, :field:`system`, + :field:`idle` are now always the first three fields. Previously on Linux, + macOS, and BSD the first three were :field:`user`, :field:`nice`, + :field:`system`. See :ref:`migration guide `. + +.. function:: cpu_percent(interval=None, percpu=False) + + Return the current system-wide CPU utilization as a percentage. + + If *interval* is > ``0.0``, measures CPU times before and after the interval + (blocking). If ``0.0`` or ``None``, returns the utilization since the last + call or module import, returning immediately. That means the first time this + is called it will return a meaningless ``0.0`` value which you are supposed + to ignore. In this case it is recommended for accuracy that this function be + called with at least ``0.1`` seconds between calls. + + If *percpu* is ``True``, returns a list of floats representing each + :term:`logical CPU`. The list is ordered by CPU index and consistent across + calls. + + This function is thread-safe. It maintains an internal map of thread IDs + (:func:`threading.get_ident`) so that independent results are returned when + called from different threads at different intervals. + + .. code-block:: pycon + + >>> import psutil + >>> # blocking + >>> psutil.cpu_percent(interval=1) + 2.0 + >>> # non-blocking (percentage since last call) + >>> psutil.cpu_percent(interval=None) + 2.9 + >>> # blocking, per-cpu + >>> psutil.cpu_percent(interval=1, percpu=True) + [5.6, 1.0] + >>> + + .. seealso:: :ref:`faq_cpu_percent` + + .. versionchanged:: 5.9.6 + the function is now thread safe. + +.. function:: cpu_times_percent(interval=None, percpu=False) + + Similar to :func:`cpu_percent`, but provides utilization percentages for each + specific CPU time. *interval* and *percpu* arguments have the same meaning as + in :func:`cpu_percent`. On Linux, :field:`guest` and :field:`guest_nice` + percentages are not accounted in :field:`user` and :field:`user_nice`. + + .. seealso:: :ref:`faq_cpu_percent` + + .. versionchanged:: 5.9.6 + function is now thread safe. + +.. function:: cpu_count(logical=True) + + Return the number of :term:`logical CPUs ` in the system (same + as :func:`os.cpu_count`), or ``None`` if undetermined. Unlike + :func:`os.cpu_count`, this is not influenced by the + :envvar:`PYTHON_CPU_COUNT` environment variable (Python 3.13+). + + If *logical* is ``False`` return the number of + :term:`physical CPUs ` only, or ``None`` if undetermined + (always ``None`` on OpenBSD and NetBSD). + + Example on a system with 2 cores + Hyper Threading: + + .. code-block:: pycon + + >>> import psutil + >>> psutil.cpu_count() + 4 + >>> psutil.cpu_count(logical=False) + 2 + + Note that this may differ from the number of CPUs the current process can + actually use (e.g. due to :term:`CPU affinity`, cgroups, or Windows processor + groups). The number of usable CPUs can be obtained with: + + .. code-block:: pycon + + >>> len(psutil.Process().cpu_affinity()) + 1 + + .. seealso:: :ref:`faq_cpu_count` + +.. function:: cpu_stats() + + Return various CPU statistics. All fields are + :term:`cumulative counters ` since boot. + + - :field:`ctx_switches`: number of :term:`context switches ` + (voluntary + involuntary). + - :field:`interrupts`: number of + :term:`hardware interrupts `. + - :field:`soft_interrupts`: number of + :term:`soft interrupts `; always set to ``0`` on Windows + and SunOS. + - :field:`syscalls`: number of system calls; always set to ``0`` on Linux. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.cpu_stats() + scpustats(ctx_switches=20455687, interrupts=6598984, soft_interrupts=2134212, syscalls=0) + +.. function:: cpu_freq(percpu=False) + + Return :field:`current`, :field:`min` and :field:`max` CPU frequencies + expressed in MHz. On Linux, :field:`current` is the real-time frequency value + (changing), on all other platforms this usually represents the nominal + "fixed" value (never changing). + + If *percpu* is ``True``, and the system supports per-CPU frequency retrieval + (Linux and FreeBSD), a list of frequencies is returned for each CPU; if not, + a list with a single element is returned. + + If :field:`min` and :field:`max` cannot be determined they are set to + ``0.0``. + + On some systems the CPU frequency cannot be determined at all (e.g. certain + virtual machines, containers or CPU architectures). In that case this returns + ``None``, or an empty list if *percpu* is ``True``. This can happen on Linux, + macOS and FreeBSD; on Windows and OpenBSD a value is always returned. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.cpu_freq() + scpufreq(current=931.42925, min=800.0, max=3500.0) + >>> psutil.cpu_freq(percpu=True) + [scpufreq(current=2394.945, min=800.0, max=3500.0), + scpufreq(current=2236.812, min=800.0, max=3500.0), + scpufreq(current=1703.609, min=800.0, max=3500.0), + scpufreq(current=1754.289, min=800.0, max=3500.0)] + + .. availability:: Linux, macOS, Windows, FreeBSD, OpenBSD. + + .. versionchanged:: 5.5.1 + added FreeBSD support. + + .. versionchanged:: 5.9.1 + added OpenBSD support. + + .. versionchanged:: 8.0.0 + on macOS ARM64 this may return ``None`` when CPU frequency data is + unavailable (e.g. on virtual machines), instead of raising. + +.. function:: getloadavg() + + Return the average system load over the last 1, 5 and 15 minutes as a tuple. + On UNIX, this relies on :func:`os.getloadavg`. On Windows, this is emulated + via a background thread that updates every 5 seconds; the first call (and for + the following 5 seconds) returns ``(0.0, 0.0, 0.0)``. The values only make + sense relative to the number of installed :term:`logical CPUs ` + (e.g. ``3.14`` on a 10-CPU system means 31.4% load). + + .. code-block:: pycon + + >>> import psutil + >>> psutil.getloadavg() + (3.14, 3.89, 4.67) + >>> psutil.cpu_count() + 10 + >>> # percentage representation + >>> [x / psutil.cpu_count() * 100 for x in psutil.getloadavg()] + [31.4, 38.9, 46.7] + +Memory +^^^^^^ + +.. function:: virtual_memory() + + Return statistics about system memory usage. All values are expressed in + bytes. + + - :field:`total`: total physical RAM. + - :field:`available`: memory that can be given instantly to processes without + the system going into :term:`swap `. This is the recommended + field for monitoring actual memory usage in a cross-platform fashion. See + :term:`available memory`. + - :field:`percent`: the percentage usage calculated as + ``(total - available) / total * 100``. + - :field:`used`: memory in use, calculated differently depending on the + platform (see the table below). It is meant for informational purposes. + Neither ``total - free`` nor ``total - available`` necessarily equals + ``used``. + - :field:`free`: memory not currently allocated to anything. This is + typically much lower than :field:`available` because the OS keeps recently + freed memory as reclaimable cache (see :field:`cached` and + :field:`buffers`) rather than zeroing it immediately. Do not use this to + check for memory pressure; use :field:`available` instead. + - :field:`active` *(Linux, macOS, BSD)*: memory currently mapped by processes + or recently accessed, held in RAM. It is unlikely to be reclaimed unless + the system is under significant memory pressure. + - :field:`inactive` *(Linux, macOS, BSD)*: memory not recently accessed. It + still holds valid data (:term:`page cache`, old allocations) but is a + candidate for reclamation or :term:`swapping `. On BSD systems + it is counted in :field:`available`. + - :field:`buffers` *(Linux, BSD)*: see :term:`buffers`. On OpenBSD + :field:`buffers` and :field:`cached` are aliases. + - :field:`cached` *(Linux, BSD, Windows)*: RAM used by the kernel to cache + file contents (data read from or written to disk). On OpenBSD + :field:`buffers` and :field:`cached` are aliases. See :term:`page cache`. + - :field:`shared` *(Linux, BSD)*: :term:`shared memory` accessible by + multiple processes simultaneously, such as in-memory ``tmpfs`` and POSIX + shared memory objects (``shm_open``). On Linux this corresponds to + ``Shmem`` in :proc:`/proc/meminfo` and is already counted within + :field:`active` / :field:`inactive`. + - :field:`slab` *(Linux)*: memory used by the kernel's internal object caches + (e.g. inode and dentry caches). The reclaimable portion (``SReclaimable``) + is already included in :field:`cached`. + - :field:`wired` *(macOS, BSD, Windows)*: memory pinned in RAM by the kernel + (e.g. kernel code and critical data structures). It can never be moved to + disk. + + Below is a table showing implementation details. All info on Linux is + retrieved from :proc:`/proc/meminfo`. On macOS via ``host_statistics64()``. + On Windows via `GetPerformanceInfo`_. + + .. list-table:: + :header-rows: 1 + :widths: 9 15 14 14 26 + :class: wide-table + + * - Field + - Linux + - macOS + - Windows + - FreeBSD + * - total + - ``MemTotal`` + - ``sysctl() hw.memsize`` + - ``PhysicalTotal`` + - ``sysctl() hw.physmem`` + * - available + - ``MemAvailable`` + - ``inactive + free`` + - ``PhysicalAvailable`` + - ``inactive + cached + free`` + * - used + - ``total - available`` + - ``active + wired`` + - ``total - available`` + - ``active + wired + cached`` + * - free + - ``MemFree`` + - ``free - speculative`` + - same as ``available`` + - ``sysctl() vm.stats.vm.v_free_count`` + * - active + - ``Active`` + - ``active`` + - + - ``sysctl() vm.stats.vm.v_active_count`` + * - inactive + - ``Inactive`` + - ``inactive`` + - + - ``sysctl() vm.stats.vm.v_inactive_count`` + * - buffers + - ``Buffers`` + - + - + - ``sysctl() vfs.bufspace`` + * - cached + - ``Cached + SReclaimable`` + - + - ``SystemCache`` + - ``sysctl() vm.stats.vm.v_cache_count`` + * - shared + - ``Shmem`` + - + - + - ``sysctl(CTL_VM/VM_METER) t_vmshr + t_rmshr`` + * - slab + - ``Slab`` + - + - + - + * - wired + - + - ``wired`` + - ``KernelNonpaged`` + - ``sysctl() vm.stats.vm.v_wire_count`` + + Example on Linux: + + .. code-block:: pycon + + >>> import psutil + >>> mem = psutil.virtual_memory() + >>> mem + svmem(total=10367352832, available=6472179712, percent=37.6, used=8186245120, free=2181107712, active=4748992512, inactive=2758115328, buffers=790724608, cached=3500347392, shared=787554304, slab=199348224) + >>> + >>> THRESHOLD = 500 * 1024 * 1024 # 500MB + >>> if mem.available <= THRESHOLD: + ... print("warning") + ... + >>> + + .. note:: + - On Linux, :field:`total`, :field:`free`, :field:`used`, :field:`shared`, + and :field:`available` match the output of the ``free`` command. + - On macOS, :field:`free`, :field:`active`, :field:`inactive`, and + :field:`wired` match ``vm_stat`` command. + - On BSD, :field:`free`, :field:`active`, :field:`inactive`, + :field:`cached`, and :field:`wired` match ``vmstat -s`` command. + - On Windows, :field:`total`, :field:`used` ("In use"), and + :field:`available` match the Task Manager (Performance > Memory tab). + + .. note:: + if you just want to know how much physical memory is left in a + cross-platform manner, rely on :field:`available` and :field:`percent` + fields. + + .. seealso:: + - :src:`scripts/meminfo.py` + - :ref:`faq_virtual_memory_available` + - :ref:`faq_used_plus_free` + + .. versionchanged:: 8.0.0 + Windows: added :field:`cached` and :field:`wired` fields. + +.. function:: swap_memory() + + Return system :term:`swap memory` statistics: + + * :field:`total`: total swap space. On Windows this is derived as + ``CommitLimit - PhysicalTotal``, representing virtual memory backed by the + page file rather than the raw page-file size. + * :field:`used`: swap space currently in use. + * :field:`free`: swap space not in use (``total - used``). + * :field:`percent`: swap usage as a percentage, calculated as + ``used / total * 100``. + * :field:`sin`: number of bytes the system has moved from disk + (:term:`swap `) back into RAM. See :term:`swap-in`. + * :field:`sout`: number of bytes the system has moved from RAM to disk + (:term:`swap `). A continuously increasing :field:`sout` rate + is a sign of memory pressure. See :term:`swap-out`. + + :field:`sin` and :field:`sout` are + :term:`cumulative counters ` since boot. Monitor their + rate of change rather than the absolute value to detect active + :term:`swapping `. On Windows both are always ``0``. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.swap_memory() + sswap(total=2097147904, used=886620160, free=1210527744, percent=42.3, sin=1050411008, sout=1906720768) + + .. seealso:: + - :src:`scripts/meminfo.py` + - :ref:`Swap activity recipe ` + + .. versionchanged:: 8.0.0 + OpenBSD: :field:`sin` / :field:`sout` are no longer set to ``0``. + +Disks +^^^^^ + +.. function:: disk_partitions(all=False) + + Return mounted disk partitions as a list. This is similar to the ``df`` + command on UNIX. When *all* is ``False``, virtual/pseudo filesystems (tmpfs, + sysfs, devtmpfs, cgroup, etc.) are excluded, keeping only physical devices + (e.g., hard disks, CD-ROM drives, USB keys). The filtering logic varies by + platform: on Linux, it checks :proc:`/proc/filesystems` for ``nodev``-flagged + types (ZFS is always included); on macOS, it checks whether the device path + exists; on SunOS and AIX, it excludes filesystems with zero total size. On + BSD, *all* is ignored and all partitions are always returned. + + * :field:`device`: the device path (e.g. "/dev/hda1"). On Windows this is the + drive letter (e.g. "C:\\"). + * :field:`mountpoint`: the mount point path (e.g. "/"). On Windows this is + the drive letter (e.g. "C:\\"). + * :field:`fstype`: the partition filesystem (e.g. "ext3" on UNIX or "NTFS" on + Windows). + * :field:`opts`: a comma-separated string indicating different mount options + for the drive/partition. Platform-dependent. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.disk_partitions() + [sdiskpart(device='/dev/sda3', mountpoint='/', fstype='ext4', opts='rw,errors=remount-ro'), + sdiskpart(device='/dev/sda7', mountpoint='/home', fstype='ext4', opts='rw')] + + .. seealso:: :src:`scripts/disk_usage.py`. + + .. versionchanged:: 5.7.4 + added :field:`maxfile` and :field:`maxpath` fields. + + .. versionchanged:: 6.0.0 + removed :field:`maxfile` and :field:`maxpath` fields. + +.. function:: disk_usage(path) + + Return disk usage statistics for the partition containing *path*. Values are + expressed in bytes and include :field:`total`, :field:`used` and + :field:`free` space, plus the :field:`percentage` usage. On UNIX, *path* must + point to a path within a **mounted** filesystem partition. This function was + later incorporated in Python 3.3 as :func:`shutil.disk_usage` (see + :bpo:`12442`). + + .. code-block:: pycon + + >>> import psutil + >>> psutil.disk_usage('/') + sdiskusage(total=21378641920, used=4809781248, free=15482871808, percent=22.5) + + .. note:: + UNIX typically reserves 5% of disk space for root. :field:`total` and + :field:`used` refer to overall space, while :field:`free` and + :field:`percent` reflect unprivileged user usage. As a result, + :field:`percent` may appear ~5% higher than expected. All values match the + ``df`` command line utility. + + .. seealso:: :src:`scripts/disk_usage.py`. + +.. function:: disk_io_counters(perdisk=False, nowrap=True) + + Return system-wide disk I/O statistics. All fields are + :term:`cumulative counters ` since boot. + + - :field:`read_count`: number of reads. + - :field:`write_count`: number of writes. + - :field:`read_bytes`: number of bytes read. + - :field:`write_bytes`: number of bytes written. + + Platform-specific fields: + + - :field:`read_time`: (all except *NetBSD* and *OpenBSD*) time spent reading + from disk (in milliseconds). + - :field:`write_time`: (all except *NetBSD* and *OpenBSD*) time spent writing + to disk (in milliseconds). + - :field:`busy_time`: (*Linux*, *FreeBSD*) time spent doing actual I/Os (in + milliseconds); see :term:`busy_time`. + - :field:`read_merged_count` (*Linux*): number of merged reads (see + `iostats doc`_). + - :field:`write_merged_count` (*Linux*): number of merged writes (see + `iostats doc`_). + + If *perdisk* is ``True``, return the same information for every physical disk + as a dictionary with partition names as the keys. + + If *nowrap* is ``True`` (default), counters that overflow and wrap to zero + are automatically adjusted so they never decrease (this can happen on very + busy or long-lived systems). ``disk_io_counters.cache_clear()`` can be used + to invalidate the *nowrap* cache. + + On diskless machines this function will return ``None`` or ``{}`` if + *perdisk* is ``True``. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.disk_io_counters() + sdiskio(read_count=8141, write_count=2431, read_bytes=290203, write_bytes=537676, read_time=5868, write_time=94922) + >>> + >>> psutil.disk_io_counters(perdisk=True) + {'sda1': sdiskio(read_count=920, write_count=1, read_bytes=2933248, write_bytes=512, read_time=6016, write_time=4), + 'sda2': sdiskio(read_count=18707, write_count=8830, read_bytes=6060, write_bytes=3443, read_time=24585, write_time=1572), + 'sdb1': sdiskio(read_count=161, write_count=0, read_bytes=786432, write_bytes=0, read_time=44, write_time=0)} + + .. note:: + On Windows, you may need to run ``diskperf -y`` command first, otherwise + this function might not detect any disks. + + .. seealso:: + - :src:`scripts/iotop.py` + - :ref:`Real-time disk I/O recipe ` + - :ref:`Real-time disk I/O percent recipe ` + +Network +^^^^^^^ + +.. function:: net_io_counters(pernic=False, nowrap=True) + + Return system-wide network I/O statistics. All fields are + :term:`cumulative counters ` since boot. + + - :field:`bytes_sent`: number of bytes sent. + - :field:`bytes_recv`: number of bytes received. + - :field:`packets_sent`: number of packets sent. + - :field:`packets_recv`: number of packets received. + - :field:`errin`: total number of errors while receiving. + - :field:`errout`: total number of errors while sending. + - :field:`dropin`: total number of incoming packets dropped at the + :term:`NIC` level. Unlike :field:`errin`, drops indicate the interface or + kernel buffer was overwhelmed. + - :field:`dropout`: total number of outgoing packets dropped (always 0 on + macOS and BSD). A non-zero and growing count is a sign of network + saturation. + + If *pernic* is ``True``, return the same information for every network + interface as a dictionary, with interface names as the keys. + + If *nowrap* is ``True`` (default), counters that overflow and wrap to zero + are automatically adjusted so they never decrease (this can happen on very + busy or long-lived systems). ``net_io_counters.cache_clear()`` can be used to + invalidate the *nowrap* cache. + + On machines with no :term:`NICs ` installed this function will return + ``None`` or ``{}`` if *pernic* is ``True``. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.net_io_counters() + snetio(bytes_sent=14508483, bytes_recv=62749361, packets_sent=84311, packets_recv=94888, errin=0, errout=0, dropin=0, dropout=0) + >>> + >>> psutil.net_io_counters(pernic=True) + {'lo': snetio(bytes_sent=547971, bytes_recv=547971, packets_sent=5075, packets_recv=5075, errin=0, errout=0, dropin=0, dropout=0), + 'wlan0': snetio(bytes_sent=13921765, bytes_recv=62162574, packets_sent=79097, packets_recv=89648, errin=0, errout=0, dropin=0, dropout=0)} + + .. seealso:: :src:`scripts/nettop.py` and :src:`scripts/ifconfig.py`. + +.. function:: net_connections(kind="inet") + + Return system-wide socket connections as a list. Each entry provides 7 + fields: + + - :field:`fd`: the socket :term:`file descriptor`; set to ``-1`` on Windows + and SunOS. + - :field:`family`: the address family, either :data:`socket.AF_INET`, + :data:`socket.AF_INET6` or :data:`socket.AF_UNIX`. + - :field:`type`: the address type, either :data:`socket.SOCK_STREAM`, + :data:`socket.SOCK_DGRAM` or :data:`socket.SOCK_SEQPACKET`. + - :field:`laddr`: the local address as a ``(ip, port)`` named tuple, or a + ``path`` for :data:`socket.AF_UNIX` sockets. + - :field:`raddr`: the remote address. When the socket is not connected, this + is either an empty tuple (``AF_INET*``) or an empty string (``""``) for + ``AF_UNIX`` sockets (see note below). + - :field:`status`: a :data:`CONN_* ` constant; + always :data:`CONN_NONE` for UDP and UNIX sockets. + - :field:`pid`: PID of the process which opened the socket. Set to ``None`` + if it can't be retrieved due to insufficient permissions (e.g. Linux). + + The *kind* parameter is a string which filters for connections matching the + following criteria: + + .. table:: + + +----------------+-----------------------------------------------------+ + | Kind value | Connections using | + +================+=====================================================+ + | ``'inet'`` | IPv4 and IPv6 | + +----------------+-----------------------------------------------------+ + | ``'inet4'`` | IPv4 | + +----------------+-----------------------------------------------------+ + | ``'inet6'`` | IPv6 | + +----------------+-----------------------------------------------------+ + | ``'tcp'`` | TCP | + +----------------+-----------------------------------------------------+ + | ``'tcp4'`` | TCP over IPv4 | + +----------------+-----------------------------------------------------+ + | ``'tcp6'`` | TCP over IPv6 | + +----------------+-----------------------------------------------------+ + | ``'udp'`` | UDP | + +----------------+-----------------------------------------------------+ + | ``'udp4'`` | UDP over IPv4 | + +----------------+-----------------------------------------------------+ + | ``'udp6'`` | UDP over IPv6 | + +----------------+-----------------------------------------------------+ + | ``'unix'`` | UNIX socket (both UDP and TCP protocols) | + +----------------+-----------------------------------------------------+ + | ``'all'`` | the sum of all the possible families and protocols | + +----------------+-----------------------------------------------------+ + + .. code-block:: pycon + + >>> import psutil + >>> psutil.net_connections() + [pconn(fd=115, family=, type=, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status=, pid=1254), + pconn(fd=117, family=, type=, laddr=addr(ip='10.0.0.1', port=43761), raddr=addr(ip='72.14.234.100', port=80), status=, pid=2987), + pconn(fd=-1, family=, type=, laddr=addr(ip='10.0.0.1', port=60759), raddr=addr(ip='72.14.234.104', port=80), status=, pid=None), + pconn(fd=-1, family=, type=, laddr=addr(ip='10.0.0.1', port=51314), raddr=addr(ip='72.14.234.83', port=443), status=, pid=None) + ...] + + .. warning:: + on Linux, retrieving some connections requires root privileges. If psutil + is not run as root, those connections are silently skipped instead of + raising :exc:`PermissionError`. That means the returned list may be + incomplete. + + .. note:: + - Linux, FreeBSD, OpenBSD: :field:`raddr` field for UNIX sockets is always + set to ``""``; this is a limitation of the OS. + - macOS and AIX: :exc:`AccessDenied` is always raised unless running as + root; this is a limitation of the OS. + - Solaris: UNIX sockets are not supported. + + .. seealso:: + + - :meth:`Process.net_connections` to get per-process connections + - :src:`scripts/netstat.py` + + .. versionchanged:: 5.9.5 + OpenBSD: retrieve :field:`laddr` path for :data:`socket.AF_UNIX` sockets + (before it was an empty string). + + .. versionchanged:: 8.0.0 + :field:`status` field is now a :class:`ConnectionStatus` enum member + instead of a plain ``str``. See :ref:`migration guide `. + +.. function:: net_if_addrs() + + Return a dict mapping each :term:`NIC` to its addresses. Interfaces may have + multiple addresses per family. Each entry includes 5 fields (addresses may be + ``None``): + + - :field:`family`: the address family, either :data:`socket.AF_INET` (IPv4), + :data:`socket.AF_INET6` (IPv6), :data:`socket.AF_UNSPEC` (a virtual or + unconfigured NIC), or :data:`AF_LINK` (a MAC address). + - :field:`address`: the primary NIC address. + - :field:`netmask`: the netmask address. + - :field:`broadcast`: the broadcast address; always ``None`` on Windows. + - :field:`ptp`: a "point to point" address (typically a VPN); always ``None`` + on Windows. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.net_if_addrs() + {'lo': [snicaddr(family=, address='127.0.0.1', netmask='255.0.0.0', broadcast='127.0.0.1', ptp=None), + snicaddr(family=, address='::1', netmask='ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', broadcast=None, ptp=None), + snicaddr(family=, address='00:00:00:00:00:00', netmask=None, broadcast='00:00:00:00:00:00', ptp=None)], + 'wlan0': [snicaddr(family=, address='192.168.1.3', netmask='255.255.255.0', broadcast='192.168.1.255', ptp=None), + snicaddr(family=, address='fe80::c685:8ff:fe45:641%wlan0', netmask='ffff:ffff:ffff:ffff::', broadcast=None, ptp=None), + snicaddr(family=, address='c4:85:08:45:06:41', netmask=None, broadcast='ff:ff:ff:ff:ff:ff', ptp=None)]} + >>> + + .. seealso:: :src:`scripts/nettop.py` and :src:`scripts/ifconfig.py`. + + .. versionchanged:: 7.0.0 + Windows: added support for :field:`broadcast` field, which is no longer + ``None``. + +.. function:: net_if_stats() + + Return a dictionary mapping each :term:`NIC` to its stats: + + - :field:`isup`: whether the NIC is up and running (bool). + - :field:`duplex`: :data:`NIC_DUPLEX_FULL`, :data:`NIC_DUPLEX_HALF` or + :data:`NIC_DUPLEX_UNKNOWN`. + - :field:`speed`: NIC speed in megabits (Mbps); ``0`` if undetermined. + - :field:`mtu`: maximum transmission unit in bytes. + - :field:`flags`: a comma-separated string of interface flags (e.g. + ``"up,broadcast,running,multicast"``); may be an empty string. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.net_if_stats() + {'eth0': snicstats(isup=True, duplex=, speed=100, mtu=1500, flags='up,broadcast,running,multicast'), + 'lo': snicstats(isup=True, duplex=, speed=0, mtu=65536, flags='up,loopback,running')} + + .. seealso:: :src:`scripts/nettop.py` and :src:`scripts/ifconfig.py`. + + .. versionchanged:: 5.7.3 + UNIX: :field:`isup` also reflects whether the :term:`NIC` is running. + + .. versionchanged:: 5.9.3 + added :field:`flags` field. + +Sensors +^^^^^^^ + +.. function:: sensors_temperatures(fahrenheit=False) + + Return hardware temperatures. Each entry represents a sensor (CPU, disk, + etc.). Values are in Celsius unless *fahrenheit* is ``True``. If unsupported, + an empty dict is returned. Each entry includes: + + - :field:`label`: string label for the sensor, if available, else ``""``. + - :field:`current`: current temperature reading (changing), or ``None`` if + unavailable. + - :field:`high`: sensor-specified high temperature threshold (fixed), or + ``None`` if unavailable. Typically indicates when hardware may start + throttling to reduce heat. + - :field:`critical`: sensor-specified critical temperature threshold (fixed), + or ``None`` if unavailable. Typically indicates when hardware considers + itself at risk; behavior may include throttling, fan ramp-up, or shutdown. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.sensors_temperatures() + {'acpitz': [shwtemp(label='', current=47.0, high=103.0, critical=103.0)], + 'asus': [shwtemp(label='', current=47.0, high=None, critical=None)], + 'coretemp': [shwtemp(label='Physical id 0', current=52.0, high=100.0, critical=100.0), + shwtemp(label='Core 0', current=45.0, high=100.0, critical=100.0), + shwtemp(label='Core 1', current=52.0, high=100.0, critical=100.0), + shwtemp(label='Core 2', current=45.0, high=100.0, critical=100.0), + shwtemp(label='Core 3', current=47.0, high=100.0, critical=100.0)]} + + .. seealso:: :src:`scripts/temperatures.py` and :src:`scripts/sensors.py`. + + .. availability:: Linux, FreeBSD + + .. versionchanged:: 5.5.0 + added FreeBSD support. + +.. function:: sensors_fans() + + Return hardware fan speeds in RPM (revolutions per minute). If unsupported, + return an empty dict. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.sensors_fans() + {'asus': [sfan(label='cpu_fan', current=3200)]} + + .. seealso:: :src:`scripts/fans.py` and :src:`scripts/sensors.py`. + + .. availability:: Linux + +.. function:: sensors_battery() + + Return battery status information. If no battery is installed or metrics + can't be determined ``None`` is returned. + + - :field:`percent`: battery power left as a percentage. + - :field:`secsleft`: a rough approximation of how many seconds are left + before the battery runs out of power. If the AC power cable is connected + this is set to :data:`POWER_TIME_UNLIMITED`. If it can't be determined it + is set to :data:`POWER_TIME_UNKNOWN`. + - :field:`power_plugged`: ``True`` if the AC power cable is connected, + ``False`` if not, or ``None`` if it can't be determined. + + .. code-block:: pycon + + >>> import psutil + >>> + >>> def secs2hours(secs): + ... mm, ss = divmod(secs, 60) + ... hh, mm = divmod(mm, 60) + ... return "%d:%02d:%02d" % (hh, mm, ss) + ... + >>> battery = psutil.sensors_battery() + >>> battery + sbattery(percent=93, secsleft=16628, power_plugged=False) + >>> print("charge = %s%%, time left = %s" % (battery.percent, secs2hours(battery.secsleft))) + charge = 93%, time left = 4:37:08 + + .. seealso:: :src:`scripts/battery.py` and :src:`scripts/sensors.py`. + + .. availability:: Linux, Windows, macOS, FreeBSD + +------------------------------------------------------------------------------- + +Other system info +^^^^^^^^^^^^^^^^^ + +.. function:: boot_time() + + Return the system boot time expressed in seconds since the epoch (seconds + since January 1, 1970, at midnight UTC). The return value is based on the + system clock, which means it can be affected by changes such as manual + adjustments or time synchronization (e.g. NTP). + + .. code-block:: pycon + + >>> import psutil, datetime + >>> psutil.boot_time() + 1389563460.0 + >>> datetime.datetime.fromtimestamp(psutil.boot_time()).strftime("%Y-%m-%d %H:%M:%S") + '2014-01-12 22:51:00' + +.. function:: users() + + Return users currently connected on the system as a list. Each entry + includes: + + - :field:`name`: the name of the user. + - :field:`terminal`: the tty or pseudo-tty associated with the user, if any, + else ``None``. + - :field:`host`: the host name associated with the entry, if any (for + example, the remote host in an SSH session), else ``None``. + - :field:`started`: the creation time as a floating point number expressed in + seconds since the epoch. + - :field:`pid`: the PID of the login process (like sshd for remote logins, + tmux, etc.). On Windows and OpenBSD this is always ``None``. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.users() + [suser(name='giampaolo', terminal='pts/2', host='localhost', started=1340737536.0, pid=1352), + suser(name='giampaolo', terminal='pts/3', host='localhost', started=1340737792.0, pid=1788)] + + .. note:: + On UNIX this reads the ``utmp`` database, and returns an empty list if + nothing maintains it, e.g. on musl libc (Alpine Linux), which doesn't + implement it. ``who`` is empty too in that case. + +------------------------------------------------------------------------------- + +Processes +--------- + +Functions +^^^^^^^^^ + +.. function:: pids() + + Return a sorted list of currently running PIDs. To iterate over all processes + and avoid race conditions :func:`process_iter` is preferred, see + :ref:`perf-process-iter`. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.pids() + [1, 2, 3, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, ..., 32498] + + .. versionchanged:: 5.6.0 + PIDs are returned in sorted order. + +.. function:: process_iter(attrs=None, ad_value=None) + + Return an iterator yielding a :class:`Process` instance for all running + processes. This should be preferred over :func:`psutil.pids` to iterate over + processes, as retrieving info is safe from race conditions. + + Every :class:`Process` instance is only created once, and then cached for the + next time :func:`psutil.process_iter` is called (if PID is still alive). + Cache can optionally be cleared via ``process_iter.cache_clear()``. + + *attrs* and *ad_value* have the same meaning as in :meth:`Process.as_dict`. + + If *attrs* is specified, :meth:`Process.as_dict` is called internally, and + the results are cached so that subsequent method calls (e.g. ``p.name()``, + ``p.status()``) return the cached values instead of issuing new system calls. + See :attr:`Process.attrs` for a list of valid *attrs* names. + + If a method raises :exc:`AccessDenied` during pre-fetch, it will return + *ad_value* (default ``None``) instead of raising. + + Processes are returned sorted by PID. + + .. code-block:: pycon + + >>> import psutil + >>> for proc in psutil.process_iter(['pid', 'name', 'username']): + ... print(proc.pid, proc.name(), proc.username()) # return cached values, never raise + ... + 1 systemd root + 2 kthreadd root + 3 ksoftirqd/0 root + ... + + All process *attrs* except slow ones: + + .. code-block:: pycon + + >>> for p in psutil.process_iter(psutil.Process.attrs - {'memory_footprint', 'memory_maps'}): + ... print(p) + + Clear internal cache: + + .. code-block:: pycon + + >>> psutil.process_iter.cache_clear() + + .. note:: + + since :class:`Process` instances are reused across calls, a subsequent + :func:`process_iter` call will overwrite or clear any previously + pre-fetched values. Do not rely on cached values from a prior iteration. + + .. seealso:: :ref:`perf-process-iter` + + .. versionchanged:: 6.0.0 + + - No longer checks whether each yielded process PID has been reused. + - Added ``psutil.process_iter.cache_clear()`` API. + + .. versionchanged:: 8.0.0 + + - When *attrs* is specified, the pre-fetched values are cached directly on + the :class:`Process` instance, so that subsequent method calls (e.g. + ``p.name()``, ``p.status()``) return the cached values instead of making + new system calls. The :attr:`Process.info` dict is deprecated in favor + of this new approach. + - Passing an empty list (``attrs=[]``) to mean "all attributes" is + deprecated; use :attr:`Process.attrs` instead. + +.. function:: pid_exists(pid) + + Check whether the given PID exists in the current process list. This is + faster than doing ``pid in psutil.pids()``, and should be preferred. + + .. seealso:: :ref:`faq_pid_exists_vs_isrunning` + +.. function:: wait_procs(procs, timeout=None, callback=None) + + Bulk operation that waits for a list of :class:`Process` instances to + terminate. Return a ``(gone, alive)`` tuple. The ``gone`` processes will have + a new ``returncode`` attribute set by :meth:`Process.wait`. + + *callback* is called with a :class:`Process` instance whenever a process + terminates. + + Returns as soon as all processes terminate or *timeout* (seconds) expires. + Unlike :meth:`Process.wait`, it does not raise :exc:`TimeoutExpired` on + timeout. + + Typical usage: + + - send SIGTERM to a list of processes + - wait a short time + - send SIGKILL to any still alive + + .. code-block:: python + + import psutil + + def on_terminate(proc): + print(f"{proc} terminated with exit code {proc.returncode}") + + procs = psutil.Process().children() + for p in procs: + p.terminate() + gone, alive = psutil.wait_procs(procs, timeout=3, callback=on_terminate) + for p in alive: + print(f"{p} is still alive, send SIGKILL") + p.kill() + +Exceptions +^^^^^^^^^^ + +.. exception:: Error() + + Base exception class. All other exceptions inherit from this one. + +.. exception:: NoSuchProcess(pid, name=None, msg=None) + + Raised by :class:`Process` class or its methods when a process with the given + *pid* is not found, no longer exists, or its PID has been reused. *name* + attribute is set only if :meth:`Process.name` was called before the process + disappeared. + + .. seealso:: :ref:`faq_no_such_process` + +.. exception:: ZombieProcess(pid, name=None, ppid=None, msg=None) + + Subclass of :exc:`NoSuchProcess`. Raised by :class:`Process` methods when + encountering a :term:`zombie process` on UNIX (Windows does not have + zombies). *name* and *ppid* attributes are set if :meth:`Process.name` or + :meth:`Process.ppid` were called before the process became a zombie. + + If you do not need to detect zombies, you can ignore this exception and just + catch :exc:`NoSuchProcess`. + + .. seealso:: :ref:`faq_zombie_process` + +.. exception:: AccessDenied(pid=None, name=None, msg=None) + + Raised by :class:`Process` methods when an action is denied due to + insufficient privileges. *name* is set if :meth:`Process.name` was called + before the exception was raised. + + .. seealso:: :ref:`faq_access_denied` + +.. exception:: TimeoutExpired(seconds, pid=None, name=None, msg=None) + + Raised by :meth:`Process.wait` method if timeout expires and the process is + still alive. *name* attribute is set if :meth:`Process.name` was previously + called. + +Process class +^^^^^^^^^^^^^ + +.. class:: Process(pid=None) + + Represents an OS process with the given *pid*. If *pid* is omitted, the + current process *pid* (:func:`os.getpid`) is used. Raises + :exc:`NoSuchProcess` if *pid* does not exist. + + On Linux, *pid* can also refer to a thread ID (the :field:`id` field returned + by :meth:`threads`). + + When calling methods of this class, always be prepared to catch + :exc:`NoSuchProcess` and :exc:`AccessDenied` exceptions. Instances can be + compared for equality and used in a :class:`set` or as :class:`dict` keys: + two instances are equal if they have the same PID and creation time. If the + creation time of either instance is unknown (e.g. on :exc:`AccessDenied`, or + for zombie processes), identity falls back on the PID alone. The same applies + on the platforms where creation time is not part of process identity (see + :ref:`faq_pid_reuse`). + + .. note:: + + This class is bound to a process via its **PID**. If the process terminates + and the OS reuses its PID, you may accidentally interact with another + process. To prevent this, use :meth:`is_running` first. Some methods (e.g., + setters and signal-related methods) perform an additional check using PID + + creation time, and will raise :exc:`NoSuchProcess` if the PID has been + reused. This check is not available on all platforms. See + :ref:`faq_pid_reuse` for details. + + .. note:: + + To fetch multiple attributes efficiently, use the :meth:`oneshot` context + manager or the :meth:`as_dict` utility method. + + .. attribute:: pid + + The process PID as a read-only property. + + .. attribute:: attrs + + A :class:`frozenset` of strings representing the valid attribute names + accepted by :meth:`as_dict` and :func:`process_iter`. It defaults to all + read-only :class:`Process` method names, minus the utility methods such as + :meth:`as_dict`, :meth:`children`, etc. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.Process.attrs + frozenset({'cmdline', 'cpu_num', 'cpu_percent', ...}) + >>> # all attrs + >>> psutil.process_iter(attrs=psutil.Process.attrs) + >>> # all attrs except 'net_connections' + >>> psutil.process_iter(attrs=psutil.Process.attrs - {"net_connections"}) + + .. versionadded:: 8.0.0 + + .. attribute:: info + + A dict containing pre-fetched process info, set by :func:`process_iter` + when called with ``attrs`` argument. Accessing this attribute is deprecated + and raises :exc:`DeprecationWarning`. Use method calls instead (e.g. + ``p.name()`` instead of ``p.info['name']``) or :func:`process_iter` + + :meth:`Process.as_dict` if you need a dict structure. + + .. seealso:: :ref:`migration guide `. + + .. deprecated:: 8.0.0 + + .. method:: oneshot() + + Context manager that speeds up retrieval of multiple process attributes. + Internally, many attributes (e.g. :meth:`name`, :meth:`ppid`, :meth:`uids`, + :meth:`create_time`, ...) share the same underlying system call; within + this context, those calls are executed once and results are cached, + avoiding redundant syscalls. + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> with p.oneshot(): + ... p.name() # actual syscall + ... p.cpu_times() # from cache + ... p.create_time() # from cache + ... p.ppid() # from cache + ... p.status() # from cache + ... + >>> + + Which methods share a syscall, and therefore get cached, depends on the + platform, so the example above is indicative. See + :ref:`perf-oneshot-methods` for the full list. + + .. seealso:: + - :doc:`performance` + - :doc:`/blog/2016/500-is-twice-as-fast` + + .. method:: name() + + The process name. On Windows the return value is cached after first call. + Not on POSIX because the process name may change. + + .. seealso:: how to :ref:`find a process by name `. + + .. method:: exe() + + The process executable as an absolute path. On some systems, if exe cannot + be determined for some internal reason (e.g. system process or path no + longer exists), this is an empty string. The return value is cached after + first call. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.Process().exe() + '/usr/bin/python3' + + .. method:: cmdline() + + The command line used to start this process, as a list of strings. The + return value is not cached because the cmdline of a process may change. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.Process().cmdline() + ['python3', 'manage.py', 'runserver'] + + .. method:: environ() + + The environment variables of the process as a dict. Note: this might not + reflect changes made after the process started. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.Process().environ() + {'LC_NUMERIC': 'it_IT.UTF-8', 'QT_QPA_PLATFORMTHEME': 'appmenu-qt5', 'IM_CONFIG_PHASE': '1', 'XDG_GREETER_DATA_DIR': '/var/lib/lightdm-data/giampaolo', 'XDG_CURRENT_DESKTOP': 'Unity', 'UPSTART_EVENTS': 'started starting', 'GNOME_KEYRING_PID': '', 'XDG_VTNR': '7', 'QT_IM_MODULE': 'ibus', 'LOGNAME': 'giampaolo', 'USER': 'giampaolo', 'PATH': '/home/giampaolo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/home/giampaolo/svn/sysconf/bin', 'LC_PAPER': 'it_IT.UTF-8', 'GNOME_KEYRING_CONTROL': '', 'GTK_IM_MODULE': 'ibus', 'DISPLAY': ':0', 'LANG': 'en_US.UTF-8', 'LESS_TERMCAP_se': '\x1b[0m', 'TERM': 'xterm-256color', 'SHELL': '/bin/bash', 'XDG_SESSION_PATH': '/org/freedesktop/DisplayManager/Session0', 'XAUTHORITY': '/home/giampaolo/.Xauthority', 'LANGUAGE': 'en_US', 'COMPIZ_CONFIG_PROFILE': 'ubuntu', 'LC_MONETARY': 'it_IT.UTF-8', 'QT_LINUX_ACCESSIBILITY_ALWAYS_ON': '1', 'LESS_TERMCAP_me': '\x1b[0m', 'LESS_TERMCAP_md': '\x1b[01;38;5;74m', 'LESS_TERMCAP_mb': '\x1b[01;31m', 'HISTSIZE': '100000', 'UPSTART_INSTANCE': '', 'CLUTTER_IM_MODULE': 'xim', 'WINDOWID': '58786407', 'EDITOR': 'vim', 'SESSIONTYPE': 'gnome-session', 'XMODIFIERS': '@im=ibus', 'GPG_AGENT_INFO': '/home/giampaolo/.gnupg/S.gpg-agent:0:1', 'HOME': '/home/giampaolo', 'HISTFILESIZE': '100000', 'QT4_IM_MODULE': 'xim', 'GTK2_MODULES': 'overlay-scrollbar', 'XDG_SESSION_DESKTOP': 'ubuntu', 'SHLVL': '1', 'XDG_RUNTIME_DIR': '/run/user/1000', 'INSTANCE': 'Unity', 'LC_ADDRESS': 'it_IT.UTF-8', 'SSH_AUTH_SOCK': '/run/user/1000/keyring/ssh', 'VTE_VERSION': '4205', 'GDMSESSION': 'ubuntu', 'MANDATORY_PATH': '/usr/share/gconf/ubuntu.mandatory.path', 'VISUAL': 'vim', 'DESKTOP_SESSION': 'ubuntu', 'QT_ACCESSIBILITY': '1', 'XDG_SEAT_PATH': '/org/freedesktop/DisplayManager/Seat0', 'LESSCLOSE': '/usr/bin/lesspipe %s %s', 'LESSOPEN': '| /usr/bin/lesspipe %s', 'XDG_SESSION_ID': 'c2', 'DBUS_SESSION_BUS_ADDRESS': 'unix:abstract=/tmp/dbus-9GAJpvnt8r', '_': '/usr/bin/python', 'DEFAULTS_PATH': '/usr/share/gconf/ubuntu.default.path', 'LC_IDENTIFICATION': 'it_IT.UTF-8', 'LESS_TERMCAP_ue': '\x1b[0m', 'UPSTART_SESSION': 'unix:abstract=/com/ubuntu/upstart-session/1000/1294', 'XDG_CONFIG_DIRS': '/etc/xdg/xdg-ubuntu:/usr/share/upstart/xdg:/etc/xdg', 'GTK_MODULES': 'gail:atk-bridge:unity-gtk-module', 'XDG_SESSION_TYPE': 'x11', 'PYTHONSTARTUP': '/home/giampaolo/.pythonstart', 'LC_NAME': 'it_IT.UTF-8', 'OLDPWD': '/home/giampaolo/svn/curio_giampaolo/tests', 'GDM_LANG': 'en_US', 'LC_TELEPHONE': 'it_IT.UTF-8', 'HISTCONTROL': 'ignoredups:erasedups', 'LC_MEASUREMENT': 'it_IT.UTF-8', 'PWD': '/home/giampaolo/svn/curio_giampaolo', 'JOB': 'gnome-session', 'LESS_TERMCAP_us': '\x1b[04;38;5;146m', 'UPSTART_JOB': 'unity-settings-daemon', 'LC_TIME': 'it_IT.UTF-8', 'LESS_TERMCAP_so': '\x1b[38;5;246m', 'PAGER': 'less', 'XDG_DATA_DIRS': '/usr/share/ubuntu:/usr/share/gnome:/usr/local/share/:/usr/share/:/var/lib/snapd/desktop', 'XDG_SEAT': 'seat0'} + + .. note:: + on macOS Big Sur this function returns something meaningful only for the + current process or in + `other specific circumstances `_. + + .. versionchanged:: 5.6.3 + added AIX support. + + .. versionchanged:: 5.7.3 + added BSD support. + + .. method:: create_time() + + The process creation time as a floating point number expressed in seconds + since the epoch (seconds since January 1, 1970, at midnight UTC). The + return value, which is cached after first call, is based on the system + clock, which means it is affected by changes such as manual adjustments or + time synchronization (e.g. NTP). + + .. code-block:: pycon + + >>> import psutil, datetime + >>> p = psutil.Process() + >>> p.create_time() + 1307289803.47 + >>> datetime.datetime.fromtimestamp(p.create_time()).strftime("%Y-%m-%d %H:%M:%S") + '2011-03-05 18:03:52' + + .. method:: as_dict(attrs=None, ad_value=None) + + Utility method returning multiple process information as a dictionary. + + If *attrs* is specified, it must be a collection of strings reflecting + available :class:`Process` class's attribute names. If not passed all + :attr:`Process.attrs` names are assumed. + + *ad_value* is the value which gets assigned to a dict key in case + :exc:`AccessDenied` or :exc:`ZombieProcess` exception is raised when + retrieving that particular process information (default ``None``). + + The ``'net_connections'`` attribute is retrieved by calling + :meth:`Process.net_connections` with ``kind="inet"``. + + Internally, :meth:`as_dict` uses :meth:`oneshot` context manager so there's + no need you use it also. + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> p.as_dict(attrs=['pid', 'name', 'username']) + {'username': 'giampaolo', 'pid': 12366, 'name': 'python'} + >>> # all attrs except slow ones + >>> p.as_dict(attrs=p.attrs - {'memory_footprint', 'memory_maps'}) + {'username': 'giampaolo', 'pid': 12366, 'name': 'python', ...} + >>> + + .. method:: ppid() + + The process parent PID. On Windows the return value is cached after the + first call. On POSIX it is not cached because it may change if the process + becomes a :term:`zombie `. See also :meth:`parent` and + :meth:`parents` methods. + + .. method:: parent() + + Utility method which returns the parent process as a :class:`Process` + object, preemptively checking whether PID has been reused. If no parent PID + is known return ``None``. See also :meth:`ppid` and :meth:`parents` + methods. + + .. method:: parents() + + Utility method which returns the parents of this process as a list of + :class:`Process` instances. If no parents are known return an empty list. + See also :meth:`ppid` and :meth:`parent` methods. + + .. method:: status() + + The current process status as a :class:`ProcessStatus` enum member. The + returned value is one of the :data:`STATUS_* ` + constants. A common use case is detecting + :term:`zombie processes ` + (``p.status() == psutil.STATUS_ZOMBIE``). + + .. versionchanged:: 8.0.0 + return value is now a :class:`ProcessStatus` enum member instead of a + plain ``str``. See :ref:`migration guide `. + + .. method:: cwd() + + The process current working directory as an absolute path. If it cannot be + determined (e.g. a system process or directory no longer exists) it returns + an empty string. + + .. versionchanged:: 5.6.4 + added support for NetBSD. + + .. method:: username() + + The name of the user that owns the process. On UNIX this is calculated by + using the :field:`real` process UID from :meth:`uids`. + + .. method:: uids() + + The :field:`real`, :field:`effective` and :field:`saved` user ID of this + process as a named tuple. This is the same as :func:`os.getresuid`, but can + be used for any process PID. + + .. availability:: UNIX + + .. method:: gids() + + The :field:`real`, :field:`effective` and :field:`saved` group ID of this + process as a named tuple. This is the same as :func:`os.getresgid`, but can + be used for any process PID. + + .. availability:: UNIX + + .. method:: terminal() + + The terminal associated with this process, if any, else ``None``. This is + similar to ``tty`` command but can be used for any process PID. + + .. availability:: UNIX + + .. method:: nice(value=None) + + Get or set process :term:`niceness ` (priority). On UNIX this is a + number which goes from ``-20`` to ``20``. The higher the nice value, the + lower the priority of the process. + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> p.nice(10) # set lower priority + >>> p.nice() # get + 10 + >>> + + On Windows *value* is one of the :ref:`*_PRIORITY_CLASS ` + constants: + + .. code-block:: pycon + + >>> p.nice(psutil.HIGH_PRIORITY_CLASS) # set higher priority + >>> p.nice() # get + + + This method was later incorporated in Python 3.3 as :func:`os.getpriority` + and :func:`os.setpriority` (see :bpo:`10784`). + + .. versionchanged:: 8.0.0 + on Windows, the return value is now a :class:`ProcessPriority` enum + member. See :ref:`migration guide `. + + .. method:: ionice(ioclass=None, value=None) + + Get or set process :term:`I/O niceness ` (priority). Called with no + arguments (get), returns a ``(ioclass, value)`` named tuple on Linux or an + *ioclass* integer on Windows. Called with *ioclass* (one of the + :ref:`IOPRIO_* ` constants), sets the I/O priority. On + Linux, an additional *value* ranging from ``0`` to ``7`` can be specified + to further adjust the priority level specified by *ioclass*. + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> if psutil.LINUX: + ... p.ionice(psutil.IOPRIO_CLASS_RT, value=7) # highest + ... else: + ... p.ionice(psutil.IOPRIO_HIGH) + ... + >>> p.ionice() # get + pionice(ioclass=, value=7) + + .. availability:: Linux, Windows + + .. versionchanged:: 5.6.2 + Windows: accept new :data:`IOPRIO_* ` constants. + + .. versionchanged:: 8.0.0 + *ioclass* is now a :class:`ProcessIOPriority` enum member. See + :ref:`migration guide `. + + .. method:: rlimit(resource, limits=None) + + Get or set process :term:`resource limits `. *resource* + must be one of the :ref:`RLIMIT_* ` constants. *limits* + is an optional ``(soft, hard)`` tuple. If provided, the method sets the + limits; if omitted, it returns the current ``(soft, hard)`` tuple. This is + the same as stdlib :func:`resource.getrlimit` and + :func:`resource.setrlimit`, but can be used for any process PID. + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> p.rlimit(psutil.RLIMIT_NOFILE, (128, 128)) # max 128 file descriptors + >>> p.rlimit(psutil.RLIMIT_FSIZE, (1024, 1024)) # max file size 1024 bytes + >>> p.rlimit(psutil.RLIMIT_FSIZE) # get current limits of ... + (1024, 1024) + + .. seealso:: :src:`scripts/procinfo.py`. + + .. availability:: Linux, FreeBSD + + .. versionchanged:: 5.7.3 + added FreeBSD support. + + .. method:: io_counters() + + Return process I/O statistics. All fields are + :term:`cumulative counters ` since process creation. + + - :field:`read_count`: number of read syscalls (e.g., :manpage:`read(2)`, + :manpage:`pread(2)`). + - :field:`write_count`: number of write syscalls (e.g., + :manpage:`write(2)`, :manpage:`pwrite(2)`). + - :field:`read_bytes`: bytes read (``-1`` on BSD). + - :field:`write_bytes`: bytes written (``-1`` on BSD). + + Linux specific: + + - :field:`read_chars` *(Linux)*: bytes read via ``read()`` and ``pread()`` + syscalls. Unlike :field:`read_bytes`, this includes tty I/O and counts + bytes regardless of whether actual disk I/O occurred (e.g. reads served + from :term:`page cache` are included). + - :field:`write_chars` *(Linux)*: bytes written via ``write()`` and + ``pwrite()`` syscalls. Same caveats as :field:`read_chars`. + + Windows specific: + + - :field:`other_count` *(Windows)*: the number of I/O operations performed + other than read and write operations. + - :field:`other_bytes` *(Windows)*: the number of bytes transferred during + operations other than read and write operations. + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> p.io_counters() + pio(read_count=454556, write_count=3456, read_bytes=110592, write_bytes=0, read_chars=769931, write_chars=203) + + .. availability:: Linux, Windows, BSD, AIX + + .. method:: num_ctx_switches() + + The number of :term:`context switches ` performed by this + process as a ``(voluntary, involuntary)`` named tuple + (:term:`cumulative counter`). + + .. note:: + (Windows, macOS) :field:`involuntary` value is always set to 0, while + :field:`voluntary` value reflects the total number of context switches + (voluntary + involuntary). This is a limitation of the OS. + + .. method:: num_fds() + + The number of :term:`file descriptors ` currently opened + by this process (non cumulative). + + .. availability:: UNIX + + .. method:: num_handles() + + The number of :term:`handles ` currently used by this process (non + cumulative). + + .. availability:: Windows + + .. method:: num_threads() + + The number of threads currently used by this process (non cumulative). + + .. method:: threads() + + Return a list of threads spawned by this process. On OpenBSD, root + privileges are required. Each entry includes: + + - :field:`id`: native thread ID assigned by the kernel. If :attr:`pid` + refers to the current process, this matches + :attr:`threading.Thread.native_id`, and can be used to reference + individual Python threads in your app. + - :field:`user_time`: time spent in user mode. + - :field:`system_time`: time spent in kernel mode. + + .. method:: cpu_times() + + Return accumulated process CPU times as + :term:`cumulative counters ` expressed in seconds. Same + as :func:`os.times`, but works for any process PID. + + - :field:`user`: time spent in user mode. + - :field:`system`: time spent in kernel mode. + - :field:`children_user`: user time of all child processes (always ``0`` on + Windows and macOS). + - :field:`children_system`: system time of all child processes (always + ``0`` on Windows and macOS). + - :field:`iowait`: (Linux) time spent waiting for blocking I/O to complete. + (:term:`iowait`). Excluded from :field:`user` and :field:`system` times + count (because the CPU is idle). + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> p.cpu_times() + pcputimes(user=0.03, system=0.67, children_user=0.0, children_system=0.0, iowait=0.08) + >>> sum(p.cpu_times()[:2]) # cumulative, excluding children and iowait + 0.70 + + .. versionchanged:: 5.6.4 + Linux: added :field:`iowait` field. + + .. method:: cpu_percent(interval=None) + + Return process CPU utilization as a percentage. Values can exceed ``100.0`` + if the process runs multiple threads on different CPUs. + + If *interval* is > ``0.0``, measures CPU times before and after the + interval (blocking). If ``0.0`` or ``None``, returns the utilization since + the last call or module import, returning immediately. That means the first + time this is called it will return a meaningless ``0.0`` value which you + are supposed to ignore. In this case it is recommended for accuracy that + this method be called with at least ``0.1`` seconds between calls. + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> # blocking + >>> p.cpu_percent(interval=1) + 2.0 + >>> # non-blocking (percentage since last call) + >>> p.cpu_percent(interval=None) + 2.9 + + .. note:: + the returned value is *not* split evenly between all available CPUs + (differently from :func:`psutil.cpu_percent`). To emulate Windows + ``taskmgr.exe`` behavior: ``p.cpu_percent() / psutil.cpu_count()``. + + .. seealso:: + - :ref:`faq_cpu_percent` + - :ref:`faq_cpu_percent_gt_100` + + .. method:: cpu_affinity(cpus=None) + + Get or set process :term:`CPU affinity` (the set of CPUs the process is + allowed to run on). If no argument is passed, return the current affinity + as a list of integers. If passed, *cpus* must be a list of CPU integers. An + empty list sets affinity to all eligible CPUs. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.cpu_count() + 4 + >>> p = psutil.Process() + >>> # get + >>> p.cpu_affinity() + [0, 1, 2, 3] + >>> # set; from now on, process will run on CPU #0 and #1 only + >>> p.cpu_affinity([0, 1]) + >>> p.cpu_affinity() + [0, 1] + >>> # reset affinity against all eligible CPUs + >>> p.cpu_affinity([]) + + .. availability:: Linux, Windows, FreeBSD + + .. method:: cpu_num() + + Return what CPU this process is currently running on. The returned number + should be ``<=`` :func:`psutil.cpu_count`. On FreeBSD certain kernel + process may return ``-1``. It may be used in conjunction with + ``psutil.cpu_percent(percpu=True)`` to observe the system workload + distributed across multiple CPUs. + + .. seealso:: :src:`scripts/cpu_distribution.py`. + + .. availability:: Linux, FreeBSD, SunOS + + .. method:: memory_info() + + Return memory information about the process. Fields vary by platform (all + values in bytes). The portable fields available on all platforms are + :field:`rss` and :field:`vms`. + + +---------+---------+----------+---------+-----+-----------------+ + | Linux | macOS | BSD | Solaris | AIX | Windows | + +=========+=========+==========+=========+=====+=================+ + | rss | rss | rss | rss | rss | rss | + +---------+---------+----------+---------+-----+-----------------+ + | vms | vms | vms | vms | vms | vms | + +---------+---------+----------+---------+-----+-----------------+ + | shared | | text | | | | + +---------+---------+----------+---------+-----+-----------------+ + | text | | data | | | | + +---------+---------+----------+---------+-----+-----------------+ + | data | | stack | | | | + +---------+---------+----------+---------+-----+-----------------+ + | | | peak_rss | | | peak_rss | + +---------+---------+----------+---------+-----+-----------------+ + | | | | | | peak_vms | + +---------+---------+----------+---------+-----+-----------------+ + + - :field:`rss`: aka :term:`RSS`. On UNIX matches the ``top`` RES column. On + Windows maps to ``WorkingSetSize``. + + - :field:`vms`: aka :term:`VMS`. On UNIX matches the ``top`` VIRT column. + On Windows maps to ``PrivateUsage`` (private committed pages only), which + differs from the UNIX definition; use :field:`virtual` from + :meth:`memory_extras` for the true virtual address space size. + + - :field:`shared` *(Linux)*: :term:`shared memory` that *could* be shared + with other processes (shared libraries, + :term:`memory-mapped files `). Counted even if no other + process is currently mapping it. Matches ``top``'s SHR column. + + - :field:`text` *(Linux, BSD)*: aka TRS (Text Resident Set). Resident + memory devoted to executable code. This memory is read-only and typically + shared across all processes running the same binary. Matches ``top``'s + CODE column. + + - :field:`data` *(Linux, BSD)*: aka DRS (Data Resident Set). On Linux this + covers the data **and** stack segments combined (from + :proc:`/proc/pid/statm`). On BSD it covers the data segment only (see + :field:`stack`). Matches ``top``'s DATA column. + + - :field:`stack` *(BSD)*: size of the process stack segment. Reported + separately from :field:`data` (unlike Linux where both are combined). + + - :field:`peak_rss` *(BSD, Windows)*: see :term:`peak_rss`. On BSD may be + ``0`` for kernel PIDs. On Windows maps to ``PeakWorkingSetSize``. + + - :field:`peak_vms` *(Windows)*: see :term:`peak_vms`. Maps to + ``PeakPagefileUsage``. + + For the full definitions of Windows fields see + `PROCESS_MEMORY_COUNTERS_EX`_. + + Example on Linux: + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> p.memory_info() + pmem(rss=15491072, vms=84025344, shared=5206016, text=2555904, data=9891840) + + .. seealso:: + - :ref:`faq_memory_rss_vs_vms` + - :ref:`faq_memory_footprint` + + .. versionchanged:: 8.0.0 (see :ref:`migration guide `) + + - *Linux*: :field:`lib` and :field:`dirty` removed (always 0 since Linux + 2.6). Deprecated aliases returning 0 and emitting + :exc:`DeprecationWarning` are kept. + - *macOS*: removed :field:`pfaults` and :field:`pageins` fields with no + backward-compatible aliases. Use :meth:`page_faults` instead. + - *Windows*: eliminated old aliases: :field:`wset` → :field:`rss`, + :field:`peak_wset` → :field:`peak_rss`, :field:`pagefile` and + :field:`private` → :field:`vms`, :field:`peak_pagefile` → + :field:`peak_vms`, :field:`num_page_faults` → :meth:`page_faults` + method. At the same time :field:`paged_pool`, :field:`nonpaged_pool`, + :field:`peak_paged_pool`, :field:`peak_nonpaged_pool` were moved to + :meth:`memory_extras`. All these old names still work but raise + :exc:`DeprecationWarning`. + - *BSD*: added :field:`peak_rss` field. + + .. method:: memory_extras() + + Return extra platform-specific memory metrics, complementing + :meth:`memory_info` (all values in bytes). + + +-------------+----------------+--------------------+ + | Linux | macOS | Windows | + +=============+================+====================+ + | peak_rss | phys_footprint | virtual | + +-------------+----------------+--------------------+ + | peak_vms | peak_footprint | peak_virtual | + +-------------+----------------+--------------------+ + | rss_anon | | paged_pool | + +-------------+----------------+--------------------+ + | rss_file | | nonpaged_pool | + +-------------+----------------+--------------------+ + | rss_shmem | | peak_paged_pool | + +-------------+----------------+--------------------+ + | swap_anon | | peak_nonpaged_pool | + +-------------+----------------+--------------------+ + | hugetlb | | | + +-------------+----------------+--------------------+ + + Linux: + + - :field:`peak_rss`: see :term:`peak_rss`. + - :field:`peak_vms`: see :term:`peak_vms`. + - :field:`rss_anon`: resident :term:`anonymous memory` (:term:`heap`, + stack, private mappings) not backed by any file. Set to 0 on Linux < 4.5. + - :field:`rss_file`: resident file-backed memory mapped from files + (:term:`shared libraries `, + :term:`memory-mapped files `). Set to 0 on Linux < 4.5. + - :field:`rss_shmem`: resident :term:`shared memory` (``tmpfs``, + ``shm_open``). ``rss_anon + rss_file + rss_shmem`` equals :field:`rss`. + Set to 0 on Linux < 4.5. + - :field:`swap_anon`: :term:`anonymous memory` currently in + :term:`swap `. Cheaper than :meth:`memory_footprint`'s + :field:`swap` (it reads :proc:`/proc/pid/status` instead of smaps) but + does not count shmem swap. Set to 0 on Linux < 2.6.34. + - :field:`hugetlb`: resident memory backed by huge pages. Set to 0 on Linux + < 4.4. + + macOS: + + - :field:`phys_footprint`: memory footprint attributed to the process, + including compressed memory. This corresponds to the main memory usage + reported by macOS Activity Monitor. Prefer it over :field:`rss`. + - :field:`peak_footprint`: the highest :field:`phys_footprint` reached over + the process lifetime. Set to 0 on macOS < 10.13. + + Windows (see `PROCESS_MEMORY_COUNTERS_EX`_): + + - :field:`virtual`: true virtual address space size, including + reserved-but-uncommitted regions (unlike :field:`vms` in + :meth:`memory_info`). + - :field:`peak_virtual`: peak virtual address space size. + - :field:`paged_pool`: kernel memory used for objects created by this + process (open file handles, registry keys, etc.) that the OS may swap to + disk under memory pressure. + - :field:`nonpaged_pool`: kernel memory used for objects that must stay in + RAM at all times (I/O request packets, device driver buffers, etc.). A + large or growing value may indicate a driver memory leak. + - :field:`peak_paged_pool`: peak paged-pool usage. + - :field:`peak_nonpaged_pool`: peak non-paged-pool usage. + + .. availability:: Linux, macOS, Windows + + .. versionadded:: 8.0.0 + + .. method:: memory_footprint() + + Return :field:`uss`, :field:`pss` and :field:`swap` memory metrics. These + give a more accurate picture of actual memory consumption than + :meth:`memory_info`. It walks the full process address space, so it is + slower than :meth:`memory_info` and may require elevated privileges. + + - :field:`uss` *(Linux, macOS, Windows)*: aka :term:`USS`; the + :term:`private memory` of the process, which would be freed if the + process were terminated right now. + + - :field:`pss` *(Linux)*: aka :term:`PSS`; shared memory divided evenly + among the processes sharing it. I.e. if a process has 10 MBs all to + itself, and 10 MBs shared with another process, its PSS will be 15 MBs. + + - :field:`swap` *(Linux)*: process memory currently in + :term:`swap `, counted per-mapping. + + Example on Linux: + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> p.memory_footprint() + pfootprint(uss=6545408, pss=6872064, swap=0) + + .. seealso:: + - :src:`scripts/procsmem.py`. + - :ref:`faq_memory_footprint` + - :doc:`/blog/2016/real-process-memory-in-python` + + .. availability:: Linux, macOS, Windows + + .. versionadded:: 8.0.0 + + .. method:: memory_full_info() + + This deprecated method returns the same information as :meth:`memory_info` + plus :meth:`memory_footprint` in a single named tuple. + + .. deprecated:: 8.0.0 + use :meth:`memory_footprint` instead. See + :ref:`migration guide `. + + .. method:: memory_percent(memtype="rss") + + Return process memory usage as a percentage of total physical memory. Same + as: + + .. code-block:: python + + Process().memory_info().rss / virtual_memory().total * 100 + + *memtype* selects which memory field to use and can be any attribute from + :meth:`memory_info`, :meth:`memory_extras`, or :meth:`memory_footprint` + (default is ``"rss"``). The divisor is always total physical memory, + regardless of *memtype*. + + .. method:: memory_maps(grouped=True) + + Return the process's :term:`memory-mapped ` file regions as + a list. Fields vary by platform (all values in bytes). + + If *grouped* is ``True``, regions with the same *path* are merged and their + numeric fields summed. If *grouped* is ``False``, each region is listed + individually; the tuple also includes *addr* (address range) and *perms* + (permission string, e.g., ``"r-xp"``). + + +---------------+---------+--------------+-----------+ + | Linux | Windows | FreeBSD | Solaris | + +===============+=========+==============+===========+ + | rss | rss | rss | rss | + +---------------+---------+--------------+-----------+ + | size | | private | anonymous | + +---------------+---------+--------------+-----------+ + | pss | | ref_count | locked | + +---------------+---------+--------------+-----------+ + | shared_clean | | shadow_count | | + +---------------+---------+--------------+-----------+ + | shared_dirty | | | | + +---------------+---------+--------------+-----------+ + | private_clean | | | | + +---------------+---------+--------------+-----------+ + | private_dirty | | | | + +---------------+---------+--------------+-----------+ + | referenced | | | | + +---------------+---------+--------------+-----------+ + | anonymous | | | | + +---------------+---------+--------------+-----------+ + | swap | | | | + +---------------+---------+--------------+-----------+ + + Linux fields (from :proc:`/proc/pid/smaps`): + + - :field:`rss`: :term:`RSS` for this mapping. + - :field:`size`: total virtual size; may far exceed :field:`rss` if parts + have never been accessed. + - :field:`pss`: :term:`PSS` for this mapping, that is :field:`rss` split + proportionally among all processes sharing it. + - :field:`shared_clean`: :term:`shared memory` not written to since loaded + (clean); can be discarded and reloaded from disk for free. + - :field:`shared_dirty`: :term:`shared memory` that has been written to + (dirty). + - :field:`private_clean`: :term:`private memory` not written to (clean). + - :field:`private_dirty`: :term:`private memory` that has been written to + (dirty); must be saved to swap before it can be freed. The key indicator + of real memory cost. + - :field:`referenced`: bytes recently accessed. + - :field:`anonymous`: :term:`anonymous memory` in this mapping + (:term:`heap`, stack). + - :field:`swap`: bytes from this mapping currently in + :term:`swap `. + + FreeBSD fields: + + - :field:`private`: :term:`private memory` in this mapping. + - :field:`ref_count`: reference count on the underlying memory object. + - :field:`shadow_count`: depth of the copy-on-write chain. + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> p.memory_maps() + [pmmap_grouped(path='/lib/x8664-linux-gnu/libutil-2.15.so', rss=32768, size=2125824, pss=32768, shared_clean=0, shared_dirty=0, private_clean=20480, private_dirty=12288, referenced=32768, anonymous=12288, swap=0), + pmmap_grouped(path='/lib/x8664-linux-gnu/libc-2.15.so', rss=3821568, size=3842048, pss=3821568, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=3821568, referenced=3575808, anonymous=3821568, swap=0), + ...] + + .. seealso:: :src:`scripts/pmap.py`. + + .. availability:: Linux, Windows, FreeBSD, SunOS + + .. versionchanged:: 5.6.0 + removed macOS support because inherently broken (see issue :gh:`1291`) + + .. method:: children(recursive=False) + + Return the children of this process as a list of :class:`Process` + instances. If *recursive* is ``True``, return all descendants. Pseudo-code + example (assuming A is this process): + + .. code-block:: none + + A ─┠+ │ + ├─ B (child) ─┠+ │ └─ X (grandchild) ─┠+ │ └─ Y (great-grandchild) + ├─ C (child) + └─ D (child) + + .. code-block:: pycon + + >>> p.children() + B, C, D + >>> p.children(recursive=True) + B, X, Y, C, D + + Note: if a process in the tree disappears (e.g., X), its descendants (Y) + won’t be returned since the reference to the parent is lost. This concept + is well illustrated by this + `unit test `_. + + .. seealso:: how to :ref:`kill a process tree `. + + .. method:: page_faults() + + Return the number of :term:`page faults ` for this process as a + ``(minor, major)`` named tuple. Both are + :term:`cumulative counters ` since process creation. + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> p.page_faults() + ppagefaults(minor=5905, major=3) + + .. versionadded:: 8.0.0 + + .. method:: open_files() + + Return regular files opened by process as a list. Each entry includes: + + - :field:`path`: the absolute file name. + - :field:`fd`: the :term:`file descriptor` number; on Windows this is + always ``-1``. + + Linux only: + + - :field:`position` (*Linux*): the file position (offset). + - :field:`mode` (*Linux*): a string indicating how the file was opened, + similarly to :func:`open` builtin *mode* argument. Possible values are + ``'r'``, ``'w'``, ``'a'``, ``'r+'`` and ``'a+'``. There's no distinction + between files opened in binary or text mode (``"b"`` or ``"t"``). + - :field:`flags` (*Linux*): the flags which were passed to the underlying + :func:`os.open` C call when the file was opened (e.g. + :data:`os.O_RDONLY`, :data:`os.O_TRUNC`, etc). + + .. code-block:: pycon + + >>> import psutil + >>> f = open('file.ext', 'w') + >>> p = psutil.Process() + >>> p.open_files() + [popenfile(path='/home/giampaolo/svn/psutil/file.ext', fd=3, position=0, mode='w', flags=32769)] + + .. warning:: + - Windows: this is not guaranteed to enumerate all file handles (see + :ref:`faq_open_files_windows`) + - NetBSD, OpenBSD: :field:`path` is always an empty string. The kernel + doesn't expose it (there's no path field in ``struct kinfo_file``). + - FreeBSD: :field:`path` can be an empty string (:gh:`595`). + + .. method:: net_connections(kind="inet") + + Same as :func:`psutil.net_connections` but for this process only (the + returned named tuples have no :field:`pid` field). The *kind* parameter and + the same limitations apply (root may be needed on some platforms). + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process(1694) + >>> p.name() + 'firefox' + >>> p.net_connections() + [pconn(fd=115, family=, type=, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status=), + pconn(fd=117, family=, type=, laddr=addr(ip='10.0.0.1', port=43761), raddr=addr(ip='72.14.234.100', port=80), status=), + pconn(fd=119, family=, type=, laddr=addr(ip='10.0.0.1', port=60759), raddr=addr(ip='72.14.234.104', port=80), status=), + pconn(fd=123, family=, type=, laddr=addr(ip='10.0.0.1', port=51314), raddr=addr(ip='72.14.234.83', port=443), status=)] + + .. method:: connections() + + Same as :meth:`net_connections` (deprecated). + + .. deprecated:: 6.0.0 + use :meth:`net_connections` instead. + + .. method:: is_running() + + Return whether the current process is running. Differently from + ``psutil.pid_exists(p.pid)``, this is reliable also in case the process is + gone and its PID reused by another process. + + If PID has been reused, this method will also remove the process from + :func:`process_iter` internal cache. + + This will return ``True`` also if the process is a :term:`zombie process` + (``p.status() == psutil.STATUS_ZOMBIE``). + + .. seealso:: + - :ref:`faq_pid_reuse` + - :ref:`faq_pid_exists_vs_isrunning` + + .. versionchanged:: 6.0.0 + automatically remove process from :func:`process_iter` internal cache if + PID has been reused by another process. + + .. method:: send_signal(sig) + + Send signal *sig* to process (see :mod:`signal` module constants), + preemptively checking whether PID has been reused. On UNIX this is the same + as ``os.kill(pid, sig)``. On Windows only ``SIGTERM``, ``CTRL_C_EVENT`` and + ``CTRL_BREAK_EVENT`` signals are supported, and ``SIGTERM`` is treated as + an alias for :meth:`kill`. + + .. seealso:: how to :ref:`kill a process tree ` + + .. method:: suspend() + + Suspend process execution with ``SIGSTOP`` signal, preemptively checking + whether PID has been reused. On UNIX this is the same as + ``os.kill(pid, signal.SIGSTOP)``. On Windows this is done by suspending all + process threads. + + .. method:: resume() + + Resume process execution with ``SIGCONT`` signal, preemptively checking + whether PID has been reused. On UNIX this is the same as + ``os.kill(pid, signal.SIGCONT)``. On Windows this is done by resuming all + process threads. + + .. method:: terminate() + + Terminate the process with ``SIGTERM`` signal, preemptively checking + whether PID has been reused. On UNIX this is the same as + ``os.kill(pid, signal.SIGTERM)``. On Windows this is an alias for + :meth:`kill`. + + .. seealso:: how to :ref:`kill a process tree `. + + .. method:: kill() + + Kill the current process by using ``SIGKILL`` signal, preemptively checking + whether PID has been reused. On UNIX this is the same as + ``os.kill(pid, signal.SIGKILL)``. On Windows this is done by using + `TerminateProcess`_. + + .. seealso:: how to :ref:`kill a process tree `. + + .. method:: wait(timeout=None) + + Wait for a process PID to terminate. The details about the return value + differ on UNIX and Windows. + + *On UNIX*: if the process terminated normally, the return value is an + integer >= 0 indicating the exit code. If the process was terminated by a + signal, returns the negated value of the signal which caused the + termination (e.g. ``-SIGTERM``). If PID is not a child of :func:`os.getpid` + (current process), it just waits until the process disappears and return + ``None``. If PID does not exist return ``None`` immediately. + + *On Windows*: always return the exit code via `GetExitCodeProcess`_. + + *timeout* is expressed in seconds. If specified, and the process is still + alive, raise :exc:`TimeoutExpired`. ``timeout=0`` can be used in + non-blocking apps: it will either return immediately or raise + :exc:`TimeoutExpired`. + + The return value is cached. To wait for multiple processes use + :func:`psutil.wait_procs`. + + .. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process(9891) + >>> p.terminate() + >>> p.wait() + + + .. seealso:: :doc:`/blog/2026/event-driven-process-waiting` + + .. note:: + + when *timeout* is not ``None`` and the platform supports it, an efficient + event-driven mechanism is used to wait for process termination: + + - Linux >= 5.3 with Python >= 3.9 uses :func:`os.pidfd_open` + + :func:`select.poll` + - macOS and other BSD variants use :func:`select.kqueue` + + ``KQ_FILTER_PROC`` + ``KQ_NOTE_EXIT`` + - Windows uses `WaitForSingleObject`_ + + If none of these mechanisms are available, the function falls back to a + busy loop (non-blocking call and short sleeps). + + Functionality also ported to the :mod:`subprocess` module in Python 3.15, + see :cpy-pr:`144047`. + + .. versionchanged:: 5.7.2 + if *timeout* is not ``None``, use efficient event-driven implementation + on Linux >= 5.3 and macOS / BSD. + + .. versionchanged:: 5.7.1 + return value is cached (instead of returning ``None``). + + .. versionchanged:: 5.7.1 + POSIX: if the signal is negative, return it as a human readable + :mod:`enum`. + + .. versionchanged:: 7.2.2 + on Linux >= 5.3 + Python >= 3.9 and macOS/BSD, use :func:`os.pidfd_open` + and :func:`select.kqueue` respectively, instead of less efficient + busy-loop polling. + +------------------------------------------------------------------------------- + +Popen class +^^^^^^^^^^^ + +.. class:: Popen(*args, **kwargs) + + Same as :class:`subprocess.Popen`, but in addition it provides all + :class:`psutil.Process` methods in a single class. For the following methods, + which are common to both classes, psutil implementation takes precedence: + :meth:`send_signal() `, + :meth:`terminate() `, + :meth:`kill() `. This is done to avoid killing another + process if its PID has been reused, fixing :bpo:`6973`. + + .. code-block:: pycon + + >>> import psutil + >>> from subprocess import PIPE + >>> + >>> p = psutil.Popen(["/usr/bin/python3", "-c", "print('hello')"], stdout=PIPE) + >>> p.name() + 'python3' + >>> p.username() + 'giampaolo' + >>> p.communicate() + ('hello\n', None) + >>> p.wait(timeout=2) + 0 + >>> + +------------------------------------------------------------------------------- + +C heap introspection +-------------------- + +The following functions provide direct access to the platform's native +:term:`heap` allocator (such as glibc's ``malloc`` on Linux or ``jemalloc`` on +BSD). They are low-level interfaces intended for detecting memory leaks in C +extensions, which are usually not revealed via standard :term:`RSS` / +:term:`VMS` metrics. These functions do not reflect Python object memory; they +operate solely on allocations made in C via :manpage:`malloc(3)`, +:manpage:`free(3)`, and related calls. + +The general idea behind these functions is straightforward: capture the state +of the :term:`heap` before and after repeatedly invoking a function implemented +in a C extension, and compare the results. If ``heap_used`` or ``mmap_used`` +grows steadily across iterations, the C code is likely retaining memory it +should be releasing. This provides an allocator-level way to spot native leaks +that Python's memory tracking misses. + +.. seealso:: + + :doc:`/blog/2025/heap-introspection-apis` + +.. tip:: + + Check out `psleak`_ project to see a practical example of how these APIs can + be used to detect memory leaks in C extensions. + +.. function:: heap_info() + + Return low-level heap statistics from the system's C allocator. On Linux, + this exposes ``uordblks`` and ``hblkhd`` fields from glibc + :manpage:`mallinfo2(3)`. + + - ``heap_used``: total number of bytes currently allocated via ``malloc()`` + (small allocations). + - ``mmap_used``: total number of bytes currently allocated via + :manpage:`mmap(2)` or via large ``malloc()`` allocations. Always set to 0 + on macOS. + - ``heap_count``: (Windows only) number of private heaps created via + ``HeapCreate()``. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.heap_info() + pheap(heap_used=5177792, mmap_used=819200) + + These fields reflect how unreleased C allocations affect the heap: + + +---------------+------------------------------------------------------------------------------------+-----------------+ + | Platform | Allocation type | Affected field | + +===============+====================================================================================+=================+ + | UNIX / glibc | small ``malloc()`` ≤128KB without ``free()`` | ``heap_used`` | + +---------------+------------------------------------------------------------------------------------+-----------------+ + | UNIX / glibc | large ``malloc()`` >128KB without ``free()`` , or ``mmap()`` without ``munmap()`` | ``mmap_used`` | + +---------------+------------------------------------------------------------------------------------+-----------------+ + | Windows | ``HeapAlloc()`` without ``HeapFree()`` | ``heap_used`` | + +---------------+------------------------------------------------------------------------------------+-----------------+ + | Windows | ``VirtualAlloc()`` without ``VirtualFree()`` | ``mmap_used`` | + +---------------+------------------------------------------------------------------------------------+-----------------+ + | Windows | ``HeapCreate()`` without ``HeapDestroy()`` | ``heap_count`` | + +---------------+------------------------------------------------------------------------------------+-----------------+ + + .. availability:: Linux with glibc, Windows, macOS, FreeBSD, NetBSD + + .. versionadded:: 7.2.0 + +.. function:: heap_trim() + + Request that the underlying allocator free any unused memory it's holding in + the :term:`heap` (typically small ``malloc()`` allocations). + + In practice, modern allocators rarely comply, so this is not a + general-purpose memory-reduction tool and won't meaningfully shrink + :term:`RSS` in real programs. Its primary value is in + **leak detection tools**. + + Calling ``heap_trim()`` before taking measurements helps reduce allocator + noise, giving you a cleaner baseline so that changes in ``heap_used`` come + from the code you're testing, not from internal allocator caching or + fragmentation. Its effectiveness depends on allocator behavior and + fragmentation patterns. + + .. availability:: Linux with glibc, Windows, macOS, FreeBSD, NetBSD + + .. versionadded:: 7.2.0 + +------------------------------------------------------------------------------- + +Windows services +---------------- + +.. function:: win_service_iter() + + Return an iterator yielding :class:`WindowsService` instances for all + installed Windows services. + + .. availability:: Windows + +.. function:: win_service_get(name) + + Get a Windows service by name, returning a :class:`WindowsService` instance. + Raise :exc:`NoSuchProcess` if no service with such name exists. + + .. availability:: Windows + +.. class:: WindowsService + + Represents a Windows service with the given *name*. This class is returned by + :func:`win_service_iter` and :func:`win_service_get` functions, and it's not + supposed to be instantiated directly. + + .. method:: name() + + The service name. This string is how a service is referenced, and can be + passed to :func:`win_service_get` to get a new :class:`WindowsService` + instance. + + .. method:: display_name() + + The service display name. The value is cached when this class is + instantiated. + + .. method:: binpath() + + The fully qualified path to the service binary/exe file as a string, + including command line arguments. + + .. method:: username() + + The name of the user that owns this service. + + .. method:: start_type() + + A string which can either be either ``'automatic'``, ``'manual'`` or + ``'disabled'``. + + .. method:: pid() + + The process PID, if any, else ``None``. This can be passed to + :class:`Process` class to control the service's process. + + .. method:: status() + + Service status as a string, which can be either ``'running'``, + ``'paused'``, ``'start_pending'``, ``'pause_pending'``, + ``'continue_pending'``, ``'stop_pending'`` or ``'stopped'``. + + .. method:: description() + + Service long description. + + .. method:: as_dict() + + Utility method retrieving all the information above as a dictionary. + + .. code-block:: pycon + + >>> import psutil + >>> list(psutil.win_service_iter()) + [, + , + , + , + ...] + >>> s = psutil.win_service_get('alg') + >>> s.as_dict() + {'binpath': 'C:\\Windows\\System32\\alg.exe', + 'description': 'Provides support for 3rd party protocol plug-ins for Internet Connection Sharing', + 'display_name': 'Application Layer Gateway Service', + 'name': 'alg', + 'pid': None, + 'start_type': 'manual', + 'status': 'stopped', + 'username': 'NT AUTHORITY\\LocalService'} + + .. availability:: Windows + +------------------------------------------------------------------------------- + +Constants +--------- + +The following enum classes group related constants, and are useful for type +annotations and introspection. The individual constants (e.g. +:data:`STATUS_RUNNING`) are also accessible directly from the psutil namespace +as aliases for the enum members, and should be preferred over accessing them +via the enum class (e.g. prefer ``psutil.STATUS_RUNNING`` over +``psutil.ProcessStatus.STATUS_RUNNING``). + +.. class:: ProcessStatus + + :class:`enum.StrEnum` collection of :data:`STATUS_* ` + constants. Returned by :meth:`Process.status`. + + .. versionadded:: 8.0.0 + +.. class:: ProcessPriority + + :class:`enum.IntEnum` collection of + :data:`*_PRIORITY_CLASS ` constants for + :meth:`Process.nice` on Windows. + + .. availability:: Windows + + .. versionadded:: 8.0.0 + +.. class:: ProcessIOPriority + + :class:`enum.IntEnum` collection of I/O priority constants for + :meth:`Process.ionice`. + + :data:`IOPRIO_CLASS_* ` on Linux, + :data:`IOPRIO_* ` on Windows. + + .. availability:: Linux, Windows + + .. versionadded:: 8.0.0 + +.. class:: ProcessRlimit + + :class:`enum.IntEnum` collection of :data:`RLIMIT_* ` + constants for :meth:`Process.rlimit`. + + .. availability:: Linux, FreeBSD + + .. versionadded:: 8.0.0 + +.. class:: ConnectionStatus + + :class:`enum.StrEnum` collection of :data:`CONN_* ` + constants. Returned in the :field:`status` field of + :func:`psutil.net_connections` and :meth:`Process.net_connections`. + + .. versionadded:: 8.0.0 + +.. class:: NicDuplex + + :class:`enum.IntEnum` collection of + :data:`NIC_DUPLEX_* ` constants. Returned in the + *duplex* field of :func:`psutil.net_if_stats`. + +.. class:: BatteryTime + + :class:`enum.IntEnum` collection of + :data:`POWER_TIME_* ` constants. May appear in the + *secsleft* field of :func:`psutil.sensors_battery`. + +.. _const-oses: + +Operating system constants +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``bool`` constants which define what platform you're on. ``True`` if on the +platform, ``False`` otherwise. + +.. data:: POSIX +.. data:: LINUX +.. data:: WINDOWS +.. data:: MACOS +.. data:: FREEBSD +.. data:: NETBSD +.. data:: OPENBSD +.. data:: BSD +.. data:: SUNOS +.. data:: AIX +.. data:: OSX + + Alias for :data:`MACOS`. + + .. deprecated:: 5.4.7 + use :data:`MACOS` instead. + +.. _const-pstatus: + +Process status constants +^^^^^^^^^^^^^^^^^^^^^^^^ + +Represent the current status of a process. Returned by :meth:`Process.status`. + +.. versionchanged:: 8.0.0 + constants are now :class:`ProcessStatus` enum members (were plain strings). + See :ref:`migration guide `. + +.. data:: STATUS_RUNNING + + The process is running or ready to run (e.g. ``while True: pass``). + +.. data:: STATUS_SLEEPING + + The process is dormant (e.g. during ``time.sleep()``) but can be woken up, + e.g. via a signal. + +.. data:: STATUS_DISK_SLEEP + + The process is waiting for disk I/O to complete. The kernel usually ignores + signals in this state to prevent data corruption. E.g. ``os.read(fd, 1024)`` + on a slow / blocked device can produce this state. + +.. data:: STATUS_STOPPED + + The process is stopped (e.g., by ``SIGSTOP`` or ``SIGTSTP``, which is sent + on Ctrl+Z) and will not run until resumed (e.g., via ``SIGCONT``). + +.. data:: STATUS_TRACING_STOP + + The process is temporarily halted because it is being inspected by a + debugger (e.g. via ``strace -p ``). + +.. data:: STATUS_ZOMBIE + + The process has finished execution and released its resources, but it + remains in the process table until the parent reaps it via ``wait()``. See + also :ref:`faq_zombie_process`. + +.. data:: STATUS_DEAD + + The process is about to disappear (final state before it is gone). + +.. data:: STATUS_WAKE_KILL + + (Linux only) A variant of :data:`STATUS_DISK_SLEEP` where the process can be + awakened by ``SIGKILL``. Used for tasks which might otherwise remain blocked + indefinitely, e.g. unresponsive network filesystems such as NFS, as in + ``open("/mnt/nfs_hung/file").read()``. + +.. data:: STATUS_WAKING + + (Linux only) A transient state right before the process becomes runnable + (:data:`STATUS_RUNNING`). + +.. data:: STATUS_PARKED + + (Linux only) A dormant state for kernel threads tied to a specific CPU. + These threads are "parked" when a CPU core is taken offline and will remain + inactive until the core is re-enabled. + +.. data:: STATUS_IDLE + + (Linux, macOS, FreeBSD) A sleep for kernel threads waiting for work. + +.. data:: STATUS_LOCKED + + (FreeBSD only) The process is blocked specifically waiting for a + kernel-level synchronization primitive (e.g. a mutex). + +.. data:: STATUS_WAITING + + (FreeBSD only) The process is waiting in a kernel sleep queue for a specific + system event to occur. + +.. data:: STATUS_SUSPENDED + + (NetBSD only) The process has been explicitly paused, similar to the stopped + state but managed by the NetBSD scheduler. + +.. _const-proc-prio: + +Process priority constants +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Represent the priority of a process on Windows (see `SetPriorityClass`_ doc). +They can be used in conjunction with :meth:`Process.nice` to get or set process +priority. + +.. availability:: Windows + +.. versionchanged:: 8.0.0 + constants are now :class:`ProcessPriority` enum members (were plain + integers). See :ref:`migration guide `. + +.. _const-prio: + +.. data:: REALTIME_PRIORITY_CLASS +.. data:: HIGH_PRIORITY_CLASS +.. data:: ABOVE_NORMAL_PRIORITY_CLASS +.. data:: NORMAL_PRIORITY_CLASS +.. data:: IDLE_PRIORITY_CLASS +.. data:: BELOW_NORMAL_PRIORITY_CLASS + +------------------------------------------------------------------------------- + +.. _const-proc-ioprio: + +Process I/O priority constants +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Represent the I/O priority class of a process (Linux and Windows only). They +can be used in conjunction with :meth:`Process.ionice` (*ioclass* argument). + +- Linux (see :manpage:`ioprio_get(2)`): + + .. data:: IOPRIO_CLASS_RT + + Highest priority. + + .. data:: IOPRIO_CLASS_BE + + Normal priority. + + .. data:: IOPRIO_CLASS_IDLE + + Lowest priority. + + .. data:: IOPRIO_CLASS_NONE + + No priority set (default; treated as :data:`IOPRIO_CLASS_BE`). + +- Windows: + + .. data:: IOPRIO_VERYLOW + .. data:: IOPRIO_LOW + .. data:: IOPRIO_NORMAL + .. data:: IOPRIO_HIGH + +.. versionchanged:: 8.0.0 + constants are now :class:`ProcessIOPriority` enum members. See + :ref:`migration guide `. + +.. _const-proc-rlimit: + +Process resource constants +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Constants for getting or setting process resource limits, to be used in in +conjunction with :meth:`Process.rlimit`. The meaning of each constant is +explained in :func:`resource.getrlimit` documentation. + +.. availability:: Linux, FreeBSD + +.. versionchanged:: 8.0.0 + these constants are now :class:`ProcessRlimit` enum members (were plain + integers). See :ref:`migration guide `. + +- Linux / FreeBSD: + + .. data:: RLIM_INFINITY + .. data:: RLIMIT_AS + .. data:: RLIMIT_CORE + .. data:: RLIMIT_CPU + .. data:: RLIMIT_DATA + .. data:: RLIMIT_FSIZE + .. data:: RLIMIT_MEMLOCK + .. data:: RLIMIT_NOFILE + .. data:: RLIMIT_NPROC + .. data:: RLIMIT_RSS + .. data:: RLIMIT_STACK + +- Linux specific: + + .. data:: RLIMIT_LOCKS + .. data:: RLIMIT_MSGQUEUE + .. data:: RLIMIT_NICE + .. data:: RLIMIT_RTPRIO + .. data:: RLIMIT_RTTIME + .. data:: RLIMIT_SIGPENDING + +- FreeBSD specific: + + .. data:: RLIMIT_SWAP + + .. versionadded:: 5.7.3 + + + .. data:: RLIMIT_SBSIZE + + .. versionadded:: 5.7.3 + + .. data:: RLIMIT_NPTS + + .. versionadded:: 5.7.3 + +.. _const-conn: + +Connections constants +^^^^^^^^^^^^^^^^^^^^^ + +:class:`enum.StrEnum` constants representing the status of a TCP connection. +Returned by :meth:`Process.net_connections` and :func:`psutil.net_connections` +(:field:`status` field). + +.. versionchanged:: 8.0.0 + constants are now :class:`ConnectionStatus` enum members (were plain + strings). See :ref:`migration guide `. + +.. data:: CONN_ESTABLISHED +.. data:: CONN_SYN_SENT +.. data:: CONN_SYN_RECV +.. data:: CONN_FIN_WAIT1 +.. data:: CONN_FIN_WAIT2 +.. data:: CONN_TIME_WAIT +.. data:: CONN_CLOSE +.. data:: CONN_CLOSE_WAIT +.. data:: CONN_LAST_ACK +.. data:: CONN_LISTEN +.. data:: CONN_CLOSING +.. data:: CONN_NONE +.. data:: CONN_DELETE_TCB (Windows) +.. data:: CONN_IDLE (Solaris) +.. data:: CONN_BOUND (Solaris) + +Hardware constants +^^^^^^^^^^^^^^^^^^ + +.. _const-aflink: + +.. data:: AF_LINK + + Identifies a MAC address associated with a network interface. Returned by + :func:`psutil.net_if_addrs` (:field:`family` field). + +.. _const-duplex: + +.. data:: NIC_DUPLEX_FULL +.. data:: NIC_DUPLEX_HALF +.. data:: NIC_DUPLEX_UNKNOWN + + Identifies whether a :term:`NIC` operates in full, half, or unknown duplex + mode. FULL allows simultaneous send/receive, HALF allows only one at a time. + Returned by :func:`psutil.net_if_stats` (:field:`duplex` field). + +.. _const-power: + +.. data:: POWER_TIME_UNKNOWN +.. data:: POWER_TIME_UNLIMITED + + Whether the remaining time of a battery cannot be determined or is unlimited. + May be assigned to :func:`psutil.sensors_battery`'s :field:`secsleft` field. + +Other constants +^^^^^^^^^^^^^^^ + +.. _const-procfs_path: + +.. data:: PROCFS_PATH + + The path of the ``/proc`` filesystem on Linux, Solaris and AIX (defaults to + ``'/proc'``). You may want to re-set this constant right after importing + psutil in case ``/proc`` is mounted elsewhere, or if you want to retrieve + information about Linux containers such as Docker, Heroku or LXC (see + `here `_ + for more info). + + It must be noted that this trick works only for APIs which rely on ``/proc`` + filesystem (e.g. memory-related APIs and many (but not all) :class:`Process` + class methods). + + .. availability:: Linux, SunOS, AIX + +Utilities +--------- + +.. function:: bytes2human(n) + + Convert *n* bytes to a human-readable string. + + .. code-block:: pycon + + >>> import psutil + >>> psutil.bytes2human(10000) + '9.8K' + >>> psutil.bytes2human(100001221) + '95.4M' + + .. versionadded:: 8.0.0 + +.. _const-version-info: + +.. data:: version_info + + A tuple to check psutil installed version. + + .. code-block:: pycon + + >>> import psutil + >>> if psutil.version_info >= (4, 5): + ... pass + +Environment variables +--------------------- + +.. envvar:: PSUTIL_DEBUG + + If set, psutil will print debug messages to stderr. This is useful for + troubleshooting internal errors or understanding the library's behavior at + a lower level. The variable is checked at import time, and affects both the + Python layer and the underlying C extension modules. It can also be toggled + programmatically at runtime via ``psutil._set_debug(True)``. + + .. code-block:: bash + + $ PSUTIL_DEBUG=1 python3 script.py + +.. envvar:: PSUTIL_BUILD_JOBS + + By default, psutil compiles its C source files in parallel, using one job per + CPU, which makes installing from source 2x to 3.6x faster. Set this variable + to change the number of jobs, or to 1 to compile serially. + + .. code-block:: bash + + $ PSUTIL_BUILD_JOBS=1 python3 -m pip install --no-binary=psutil psutil + + .. versionadded:: 8.0.0 + +.. ============================================================================ + +.. _`iostats doc`: https://www.kernel.org/doc/Documentation/iostats.txt +.. _`psleak`: https://github.com/giampaolo/psleak +.. _`GetExitCodeProcess`: https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getexitcodeprocess +.. _`GetPerformanceInfo`: https://learn.microsoft.com/en-us/windows/win32/api/psapi/nf-psapi-getperformanceinfo +.. _`PROCESS_MEMORY_COUNTERS_EX`: https://learn.microsoft.com/en-us/windows/win32/api/psapi/ns-psapi-process_memory_counters_ex +.. _`SetPriorityClass`: https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-setpriorityclass +.. _`TerminateProcess`: https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-terminateprocess +.. _`WaitForSingleObject`: https://learn.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-waitforsingleobject diff --git a/docs/blog.rst b/docs/blog.rst new file mode 100644 index 0000000000..ce69700845 --- /dev/null +++ b/docs/blog.rst @@ -0,0 +1,7 @@ +.. + Placeholder for the ``Blog `` toctree entry in index.rst. + Content is unused: ablog overrides blog.html at HTML-generation + time via docs/_templates/ablog/collection.html. + +Blog +==== diff --git a/docs/blog/2014/announcing-20.rst b/docs/blog/2014/announcing-20.rst new file mode 100644 index 0000000000..726915b722 --- /dev/null +++ b/docs/blog/2014/announcing-20.rst @@ -0,0 +1,85 @@ +.. post:: 2014-03-10 + :tags: api-design, compatibility, release + :author: Giampaolo Rodola + :exclude: + + A major rewrite with breaking API changes + +Announcing psutil 2.0 +===================== + +psutil 2.0 is out. This is a major rewrite and reorganization of both the +Python and C extension modules. It costed me four months of work and more than +**22,000 lines** (the diff against old 1.2.1). Many of the changes are not +backward compatible; I'm sure this will cause some pain, but I think it's for +the better and needed to be done. + +API changes +----------- + +I already wrote a detailed :doc:`blog post ` about +this, so use that as the official reference on how to port your code. + +RST documentation +----------------- + +I've never been happy with the old doc hosted on Google Code. The markup +language provided by Google is pretty limited, plus it's not under revision +control. The new doc is more detailed, uses reStructuredText as the markup +language, lives in the same code repository as psutil, and is hosted on the +excellent Read the Docs: http://psutil.readthedocs.org/ + +Physical CPUs count +------------------- + +You're now able to distinguish between :term:`logical ` and +:term:`physical ` CPUs. The full story is in :gh:`427`. + +.. code-block:: pycon + + >>> psutil.cpu_count() # logical + 4 + >>> psutil.cpu_count(logical=False) # physical cores only + 2 + +Process instances are hashable +------------------------------ + +:class:`psutil.Process` instances can now be compared for equality and used in +sets and dicts. The most useful application is diffing process snapshots: + +.. code-block:: pycon + + >>> before = set(psutil.process_iter()) + >>> # ... some time passes ... + >>> after = set(psutil.process_iter()) + >>> new_procs = after - before # processes spawned in between + +Equality is not just PID-based. It also includes the process creation time, so +a :class:`~psutil.Process` whose PID got reused by the kernel won't be mistaken +for the original. The full story is in :gh:`452`. + +Speedups +-------- + +* :gh:`477`: :meth:`Process.cpu_percent` is about 30% faster. +* :gh:`478`: [Linux] almost all APIs are about 30% faster on Python 3.X. + +Other improvements and bugfixes +------------------------------- + +* :gh:`424`: published Windows installers for Python 3.X 64-bit. +* :gh:`447`: the :func:`psutil.wait_procs` ``timeout`` parameter is now + optional. +* :gh:`459`: a + `Makefile `_ is now + available for running tests and other repetitive tasks (also on Windows). +* :gh:`463`: the ``timeout`` parameter of ``cpu_percent*`` functions defaults + to 0.0, because the previous default was a common source of slowdowns. +* :gh:`340`: [Windows] :meth:`Process.open_files` no longer hangs. +* :gh:`448`: [Windows] fixed a memory leak affecting :meth:`Process.children` + and :meth:`Process.ppid`. +* :gh:`461`: namedtuples are now pickle-able. +* :gh:`474`: [Windows] :meth:`Process.cpu_percent` is no longer capped at 100%. + +See the :ref:`changelog <200>` for a full list of changes. diff --git a/docs/blog/2014/porting-to-20.rst b/docs/blog/2014/porting-to-20.rst new file mode 100644 index 0000000000..6221714f57 --- /dev/null +++ b/docs/blog/2014/porting-to-20.rst @@ -0,0 +1,310 @@ +.. post:: 2014-01-11 + :tags: api-design, compatibility + :author: Giampaolo Rodola + :exclude: + + A migration guide for the breaking API changes in psutil 2.0 + +Porting your code to psutil 2.0 +=============================== + +This blog post is going to be about psutil 2.0, a major release in which I +decided to reorganize the existing API for the sake of consistency. At the time +of writing, psutil 2.0 is still under development, and the intent of this blog +post is to serve as an official reference that describes how you should port +your existing code base. In doing so, I will also explain why I decided to make +these changes. Even though many APIs will still be available as aliases +pointing to the newer ones, the overall changes are numerous and many of them +are not backward compatible. I'm sure many people will be sorely bitten, but I +think this is for the better and it needed to be done, hopefully for the first +and last time. + +Module constants turned into functions +-------------------------------------- + +**What changed** + ++---------------------------+-----------------------------------+ +| Old name | Replacement | ++===========================+===================================+ +| ``psutil.BOOT_TIME`` | ``psutil.boot_time()`` | ++---------------------------+-----------------------------------+ +| ``psutil.NUM_CPUS`` | ``psutil.cpu_count()`` | ++---------------------------+-----------------------------------+ +| ``psutil.TOTAL_PHYMEM`` | ``psutil.virtual_memory().total`` | ++---------------------------+-----------------------------------+ + +**Why I did it** + +I already talked about this more extensively in the previous +`Making constants part of your API is evil `__ +blog post. In short: other than introducing unnecessary slowdowns, calculating +a module-level constant at import time is dangerous because if something goes +wrong the whole app will crash. Also, the represented values may be subject to +change (think about the system clock), but the constant cannot be updated. +Thanks to this hack, accessing the old constants still works and produces a +:exc:`DeprecationWarning`. + +Renamed module functions +------------------------ + +**What changed** + ++----------------------------+------------------------+ +| Old name | Replacement | ++============================+========================+ +| ``psutil.get_boot_time()`` | ``psutil.boot_time()`` | ++----------------------------+------------------------+ +| ``psutil.get_pid_list()`` | ``psutil.pids()`` | ++----------------------------+------------------------+ +| ``psutil.get_users()`` | ``psutil.users()`` | ++----------------------------+------------------------+ + +**Why I did it** + +They were the only module-level functions with a ``get_`` prefix. None of the +others had one. + +Renamed Process class methods +----------------------------- + +All methods lost their ``get_`` and ``set_`` prefixes. A single method can now +be used for both getting and setting (if a value is passed). Assuming +``p = psutil.Process()``: + ++------------------------------+--------------------------+ +| Old name | Replacement | ++==============================+==========================+ +| ``p.get_children()`` | ``p.children()`` | ++------------------------------+--------------------------+ +| ``p.get_connections()`` | ``p.connections()`` | ++------------------------------+--------------------------+ +| ``p.get_cpu_affinity()`` | ``p.cpu_affinity()`` | ++------------------------------+--------------------------+ +| ``p.get_cpu_percent()`` | ``p.cpu_percent()`` | ++------------------------------+--------------------------+ +| ``p.get_cpu_times()`` | ``p.cpu_times()`` | ++------------------------------+--------------------------+ +| ``p.get_io_counters()`` | ``p.io_counters()`` | ++------------------------------+--------------------------+ +| ``p.get_ionice()`` | ``p.ionice()`` | ++------------------------------+--------------------------+ +| ``p.get_memory_info()`` | ``p.memory_info()`` | ++------------------------------+--------------------------+ +| ``p.get_ext_memory_info()`` | ``p.memory_info_ex()`` | ++------------------------------+--------------------------+ +| ``p.get_memory_maps()`` | ``p.memory_maps()`` | ++------------------------------+--------------------------+ +| ``p.get_memory_percent()`` | ``p.memory_percent()`` | ++------------------------------+--------------------------+ +| ``p.get_nice()`` | ``p.nice()`` | ++------------------------------+--------------------------+ +| ``p.get_num_ctx_switches()`` | ``p.num_ctx_switches()`` | ++------------------------------+--------------------------+ +| ``p.get_num_fds()`` | ``p.num_fds()`` | ++------------------------------+--------------------------+ +| ``p.get_num_threads()`` | ``p.num_threads()`` | ++------------------------------+--------------------------+ +| ``p.get_open_files()`` | ``p.open_files()`` | ++------------------------------+--------------------------+ +| ``p.get_rlimit()`` | ``p.rlimit()`` | ++------------------------------+--------------------------+ +| ``p.get_threads()`` | ``p.threads()`` | ++------------------------------+--------------------------+ +| ``p.getcwd()`` | ``p.cwd()`` | ++------------------------------+--------------------------+ + +...as for ``set_*`` methods: + ++--------------------------+-------------------------------------+ +| Old name | Replacement | ++==========================+=====================================+ +| ``p.set_cpu_affinity()`` | ``p.cpu_affinity(cpus)`` | ++--------------------------+-------------------------------------+ +| ``p.set_ionice()`` | ``p.ionice(ioclass, value=None)`` | ++--------------------------+-------------------------------------+ +| ``p.set_nice()`` | ``p.nice(value)`` | ++--------------------------+-------------------------------------+ +| ``p.set_rlimit()`` | ``p.rlimit(resource, limits=None)`` | ++--------------------------+-------------------------------------+ + +**Why I did it** + +I wanted to be consistent with system-wide module-level functions, which have +no ``get_`` prefix. After I got rid of the ``get_`` prefixes, removing ``set_`` +too seemed natural and helped reduce the number of methods. + +Process properties are now methods +---------------------------------- + +**What changed** + +Assuming ``p = psutil.Process()``: + ++-------------------+---------------------+ +| Old name | Replacement | ++===================+=====================+ +| ``p.cmdline`` | ``p.cmdline()`` | ++-------------------+---------------------+ +| ``p.create_time`` | ``p.create_time()`` | ++-------------------+---------------------+ +| ``p.exe`` | ``p.exe()`` | ++-------------------+---------------------+ +| ``p.gids`` | ``p.gids()`` | ++-------------------+---------------------+ +| ``p.name`` | ``p.name()`` | ++-------------------+---------------------+ +| ``p.parent`` | ``p.parent()`` | ++-------------------+---------------------+ +| ``p.ppid`` | ``p.ppid()`` | ++-------------------+---------------------+ +| ``p.status`` | ``p.status()`` | ++-------------------+---------------------+ +| ``p.uids`` | ``p.uids()`` | ++-------------------+---------------------+ +| ``p.username`` | ``p.username()`` | ++-------------------+---------------------+ + +**Why I did it** + +Different reasons: + +* Having a mixed API that uses both properties and methods for no particular + reason is confusing and messy, because you don't know whether to use ``()`` + or not. +* A property is usually expected not to perform heavy computations internally, + whereas psutil invokes a function every time it is accessed. This has two + drawbacks: + + * You may get an exception just by accessing the property (e.g. ``p.name`` + may raise :exc:`NoSuchProcess` or :exc:`AccessDenied`). + * You may erroneously think properties are cached, but this is true only for + ``name``, ``exe``, and ``create_time``. + +CPU percent intervals +--------------------- + +**What changed** + +The timeout parameter of ``cpu_percent*`` functions now defaults to 0.0 instead +of 0.1. The functions affected are: + +* :meth:`Process.cpu_percent` +* :func:`psutil.cpu_percent` +* :func:`psutil.cpu_times_percent` + +**Why I did it** + +I originally set 0.1 as the default timeout because you need to wait some time +in order to get a meaningful percent value. Having an API that "sleeps" by +default is risky, though, because it's easy to forget it does so. That is +particularly problematic when calling :meth:`Process.cpu_percent` for all +processes: it's very easy to forget to specify ``timeout=0``, resulting in +dramatic slowdowns that are hard to spot. For example, this code snippet might +take a variable number of seconds to complete depending on the number of active +processes: + +.. code-block:: pycon + + >>> # this will be slow + >>> for p in psutil.process_iter(): + ... print(p.cpu_percent()) + +Migration strategy +------------------ + +Except for :class:`Process` properties (:meth:`Process.name`, +:meth:`Process.exe`, :meth:`Process.cmdline`, etc.), all the old APIs are still +available as aliases pointing to the newer names and raising +:exc:`DeprecationWarning`. psutil will be very clear on what you should use +instead of the deprecated API, as long as you start the interpreter with the +``-Wd`` option. This will enable deprecation warnings, which were +`silenced in Python 2.7 `__ (IMHO, from a +developer standpoint this was a bad decision). + +:: + + giampaolo@ubuntu:/tmp$ python -Wd + Python 2.7.3 (default, Sep 26 2013, 20:03:06) + [GCC 4.6.3] on linux2 + Type "help", "copyright", "credits" or "license" for more information. + >>> import psutil + >>> psutil.get_pid_list() + __main__:1: DeprecationWarning: psutil.get_pid_list is deprecated; use psutil.pids() instead + [1, 2, 3, 6, 7, 13, ...] + >>> + >>> + >>> p = psutil.Process() + >>> p.get_cpu_times() + __main__:1: DeprecationWarning: get_cpu_times() is deprecated; use cpu_times() instead + pcputimes(user=0.08, system=0.03) + >>> + +If you have a solid test suite, you can run tests and fix the warnings one by +one. As for the Process properties that were turned into methods, it's more +difficult because, whereas psutil 1.2.1 returns the actual value, psutil 2.0.0 +returns the bound method: + +.. code-block:: pycon + + # psutil 1.2.1 + >>> psutil.Process().name + 'python' + >>> + + # psutil 2.0.0 + >>> psutil.Process().name + + >>> + +What I would recommend, if you want to drop support for 1.2.1, is to grep for +``".name"``, ``".exe"``, etc. and just replace them with ``".exe()"`` and +``".name()"`` one by one. If, on the other hand, you want to write code that +works with both versions, I see two possibilities: + +* #1 check version info, like this: + +.. code-block:: pycon + + >>> PSUTIL2 = psutil.version_info >= (2, 0) + >>> p = psutil.Process() + >>> name = p.name() if PSUTIL2 else p.name + >>> exe = p.exe() if PSUTIL2 else p.exe + +* #2 get rid of all ``".name"``, ``".exe"`` occurrences you have in your code + and use :meth:`Process.as_dict` instead: + +.. code-block:: pycon + + >>> p = psutil.Process() + >>> pinfo = p.as_dict(attrs=["name", "exe"]) + >>> pinfo + {'exe': '/usr/bin/python2.7', 'name': 'python'} + >>> name = pinfo['name'] + >>> exe = pinfo['exe'] + +New features introduced in 2.0.0 +-------------------------------- + +psutil 2.0.0 is not only about code breakage. I also had the chance to +integrate a bunch of interesting features. + +* :gh:`427`: you're now able to distinguish between the number of + :term:`logical ` and :term:`physical ` CPUs: + +.. code-block:: pycon + + >>> psutil.cpu_count() # logical + 4 + >>> psutil.cpu_count(logical=False) # physical cores only + 2 + +* :gh:`452`: :class:`Process` instances are now hashable and can be checked for + equality. That means you can use :class:`Process` objects with sets + (finally!). +* :gh:`447`: the ``timeout`` parameter of :func:`psutil.wait_procs` is now + optional. +* :gh:`461`: functions returning namedtuples are now picklable. +* :gh:`459`: a Makefile is now available to automate repetitive tasks such as + build, install, running tests, etc. There's also a make.bat for Windows. +* Introduced the ``unittest2`` module as a requirement for running tests. diff --git a/docs/blog/2015/openbsd-support.rst b/docs/blog/2015/openbsd-support.rst new file mode 100644 index 0000000000..e8ee38d61f --- /dev/null +++ b/docs/blog/2015/openbsd-support.rst @@ -0,0 +1,73 @@ +.. post:: 2015-11-25 + :tags: bsd, new-platform, community, release + :author: Giampaolo Rodola + :exclude: + + psutil 3.3.0 now runs on OpenBSD, sharing most of the implementation with FreeBSD + +OpenBSD support +=============== + +Starting from version 3.3.0 (released just now) psutil officially supports +OpenBSD. This was contributed by `Landry Breuil `__ +and myself in :pr:`615`. + +Differences with FreeBSD +------------------------ + +As expected, the OpenBSD implementation is very similar to FreeBSD's, so I +merged most of it into a single C file +(`_psutil_bsd.c `__), +with 2 separate files +(`freebsd.c `__, +`openbsd.c `__) +for the parts that differ. Here are the functional differences with FreeBSD: + +* :meth:`Process.memory_maps` is not implemented. The kernel provides the + necessary pieces but I haven't done this yet (hopefully later). +* :meth:`Process.num_ctx_switches`'s :field:`involuntary` field is always 0. + `kinfo_proc `__ + provides this info but it is always set to 0. +* :meth:`Process.cpu_affinity` (get and set) is not supported. +* :meth:`Process.exe` is determined by inspecting the command line, so it may + not always be available (returns ``None``). +* :func:`psutil.swap_memory` :field:`sin` and :field:`sout` + (:term:`swap in ` and :term:`swap out `) values are not + available, and are therefore always set to 0. +* :func:`psutil.cpu_count` ``(logical=False)`` always returns ``None``. + +As with FreeBSD, :meth:`Process.open_files` can't return file paths (FreeBSD +can sometimes). Otherwise everything is there and I'm satisfied with the +result. + +Considerations about BSD platforms +---------------------------------- + +psutil has supported FreeBSD since the beginning (year 2009). At the time, it +made sense to prefer it as the preferred BSD variant as it's the most +`most popular `__. + +Compared to FreeBSD, OpenBSD appears to be more "minimal", both in terms of +kernel facilities and the number of CLI tools available. One thing I +particularly appreciate about FreeBSD is that the source code for all CLI tools +is available under ``/usr/src``, which was a big help when implementing psutil +APIs. + +OpenBSD source code is `also available `__, but +it uses CVS and I am not sure it includes the source code for all CLI tools. + +There are still two more BSD variants worth supporting: NetBSD and DragonFlyBSD +(in this order). About a year ago, someone provided a patch (:gh:`429`) adding +basic NetBSD support, so that will likely happen sooner or later. + +Other enhancements available in this release +-------------------------------------------- + +The only other enhancement is :gh:`558`, which allows specifying a different +location for the /proc filesystem on Linux. + +Discussion +---------- + +* `Reddit `__ +* `Hacker News `__ diff --git a/docs/blog/2015/reimplementing-ifconfig-in-python.rst b/docs/blog/2015/reimplementing-ifconfig-in-python.rst new file mode 100644 index 0000000000..85a7940646 --- /dev/null +++ b/docs/blog/2015/reimplementing-ifconfig-in-python.rst @@ -0,0 +1,126 @@ +.. post:: 2015-06-13 + :tags: personal, new-api, compatibility, featured, release + :author: Giampaolo Rodola + :exclude: + + psutil 3.0 introduces :func:`~psutil.net_if_addrs` and :func:`~psutil.net_if_stats` + +Reimplementing ifconfig in Python +================================= + +Here we are. It's been a long time since my last blog post and my last psutil +release. The reason? I've been travelling! I mean... a lot. I've spent 3 months +in Berlin, 3 weeks in Japan and 2 months in New York City. While I was there I +finally had the chance to meet my friend +`Jay Loden `__ in person. We originally +started working on psutil together +`7 years ago `__. + +.. image:: https://gmpy.dev/images/me-with-jay.jpg + :alt: Jay and I + :width: 750px + :target: https://gmpy.dev/images/me-with-jay.jpg + +Back then I didn't know any C (and I'm still a terrible C developer), so he was +crucial in developing the initial psutil skeleton, including macOS and Windows +support. Needless to say that this release builds on that work. + +net_if_addrs() +-------------- + +We're now able to list network interface addresses similarly to the +``ifconfig`` command on UNIX: + +.. code-block:: pycon + + >>> import psutil + >>> from pprint import pprint + >>> pprint(psutil.net_if_addrs()) + {'ethernet0': [snic(family=, + address='10.0.0.4', + netmask='255.0.0.0', + broadcast='10.255.255.255'), + snic(family=, + address='9c:eb:e8:0b:05:1f', + netmask=None, + broadcast='ff:ff:ff:ff:ff:ff')], + 'localhost': [snic(family=, + address='127.0.0.1', + netmask='255.0.0.0', + broadcast='127.0.0.1'), + snic(family=, + address='00:00:00:00:00:00', + netmask=None, + broadcast='00:00:00:00:00:00')]} + +This is limited to ``AF_INET`` (IPv4), ``AF_INET6`` (IPv6) and ``AF_LINK`` +(Ethernet) address families. If you want something more powerful (e.g. +``AF_BLUETOOTH``) you can take a look at the +`netifaces `__ extension. If you want +to see how this is implemented, here's the code for POSIX and Windows: + +* `POSIX `__ +* `Windows `__ + +net_if_stats() +-------------- + +This new function returns information about network interface cards: + +.. code-block:: pycon + + >>> import psutil + >>> from pprint import pprint + >>> pprint(psutil.net_if_stats()) + {'ethernet0': snicstats(isup=True, + duplex=, + speed=100, + mtu=1500), + 'localhost': snicstats(isup=True, + duplex=, + speed=0, + mtu=65536)} + +The implementation on each platform: + +* `Windows `__ +* `Linux `__ +* `macOS & FreeBSD `__ +* `SunOS `__ + +Also in 3.0 +----------- + +Beyond the network-interface APIs, psutil 3.0 ships a few other notable +changes. + +Several integer/string constants (``IOPRIO_CLASS_*``, ``NIC_DUPLEX_*``, +``*_PRIORITY_CLASS``) now return :mod:`enum` values on Python 3.4+. + +Support for :term:`zombie processes ` on UNIX was broken. +Covered in a :doc:`separate post `. + +All aliases deprecated in the +:doc:`psutil 2.0 porting guide ` (January 2014) are +gone. + +For a full list of changes see the :ref:`changelog <300>`. + +Final words +----------- + +I must say I'm pretty satisfied with how psutil is evolving and with the +enjoyment I still get every time I work on it. It now gets almost +`800,000 downloads a month `__, which is +quite remarkable for a Python library. + +At this point, I consider psutil almost "complete" feature-wise, meaning I'm +starting to run out of ideas for what to add next (see +`TODO `__). Going +forward, development will likely focus on supporting more exotic platforms +(OpenBSD :gh:`562`, NetBSD :pr:`557`, Android :gh:`355`). + +There have also been discussions on the python-ideas mailing list about +`including psutil in the Python stdlib `__, +but even if that happens, it's still a long way off, as it would require a +significant time investment that I currently don't have. diff --git a/docs/blog/2015/zombie-processes.rst b/docs/blog/2015/zombie-processes.rst new file mode 100644 index 0000000000..64c12ca6df --- /dev/null +++ b/docs/blog/2015/zombie-processes.rst @@ -0,0 +1,88 @@ +.. post:: 2015-06-13 + :tags: api-design, compatibility + :author: Giampaolo Rodola + :exclude: + + Proper handling of zombies via new :exc:`ZombieProcess` exception + +Proper zombie process handling +============================== + +This is part of the psutil 3.0 release (see the full +:doc:`release notes `). + +Except on Linux and Windows (which does not have them), support for +:term:`zombie processes ` was broken. The full story is in +:gh:`428`. + +The problem +----------- + +Say you create a :term:`zombie process` and instantiate a +:class:`~psutil.Process` for it: + +.. code-block:: python + + import os, time + + def create_zombie(): + pid = os.fork() # the zombie + if pid == 0: + os._exit(0) # child exits immediately + else: + time.sleep(1000) # parent does NOT call wait() + + pid = create_zombie() + p = psutil.Process(pid) + +Up until psutil 2.X, every time you tried to query it you'd get a +:exc:`NoSuchProcess` exception: + +.. code-block:: pycon + + >>> p.name() + File "psutil/__init__.py", line 374, in _init + raise NoSuchProcess(pid, None, msg) + psutil.NoSuchProcess: no process found with pid 123 + +This was misleading, because the PID technically still existed: + +.. code-block:: pycon + + >>> psutil.pid_exists(p.pid) + True + +Depending on the platform, some process information could still be retrieved: + +.. code-block:: pycon + + >>> p.cmdline() + ['python'] + +Worst of all, :func:`psutil.process_iter` didn't return zombies at all. That +was a real problem, because identifying them is a legitimate use case: a zombie +usually indicates a bug where a parent process spawns a child, kills it, but +never calls ``wait()`` to reap it. + +What changed +------------ + +* A new :exc:`ZombieProcess` exception is raised whenever a process cannot be + queried because it is a zombie. +* It replaces :exc:`NoSuchProcess`, which was incorrect and misleading. +* :exc:`ZombieProcess` inherits from :exc:`NoSuchProcess`, so existing code + keeps working. +* :func:`psutil.process_iter` now correctly includes zombie processes, so you + can reliably identify them: + +.. code-block:: python + + import psutil + + zombies = [] + for p in psutil.process_iter(): + try: + if p.status() == psutil.STATUS_ZOMBIE: + zombies.append(p) + except psutil.NoSuchProcess: + pass diff --git a/docs/blog/2016/500-is-twice-as-fast.rst b/docs/blog/2016/500-is-twice-as-fast.rst new file mode 100644 index 0000000000..32e49a362b --- /dev/null +++ b/docs/blog/2016/500-is-twice-as-fast.rst @@ -0,0 +1,204 @@ +.. post:: 2016-11-06 + :tags: performance, new-api, featured, release + :author: Giampaolo Rodola + :category: featured + :exclude: + + New :meth:`Process.oneshot`, caching shared syscalls and halving the cost of multi-field queries + +Making psutil twice as fast +=========================== + +Starting from psutil 5.0.0 you can query multiple :class:`Process` fields +around twice as fast as before (see :gh:`799` and :meth:`Process.oneshot`). It +took 7 months, 108 commits, and a massive refactoring of psutil internals +(:pr:`937`), and I think it's one of the best improvements ever shipped in a +psutil release. + +The problem +----------- + +How process information is retrieved varies by OS. Sometimes it means reading a +file in /proc (Linux), other times calling C (Windows, BSD, macOS, SunOS), but +it's always done differently. Psutil abstracts this away: you call +:meth:`Process.name` without worrying about what happens under the hood or +which OS you're on. + +Internally, multiple pieces of process info (e.g. :meth:`Process.name`, +:meth:`Process.ppid`, :meth:`Process.uids`, :meth:`Process.create_time`) are +fetched by the same syscall. On Linux we read :proc:`/proc/pid/stat` to get the +process name, terminal, CPU times, creation time, status and parent PID, but +only one value is returned: the others are discarded. On Linux this code reads +:proc:`/proc/pid/stat` 6 times: + +.. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> p.name() + >>> p.cpu_times() + >>> p.create_time() + >>> p.ppid() + >>> p.status() + >>> p.terminal() + +On BSD most process metrics can be fetched with a single ``sysctl()``, yet +psutil was invoking it for each process method (e.g. see +`here `__ +and +`here `__). + +Do it in one shot +----------------- + +It's clear that this approach is inefficient, especially in tools like top or +htop, where process info is continuously fetched in a loop. psutil 5.0.0 +introduces a new :meth:`Process.oneshot` context manager. Inside it, the +internal routine runs once (in the example, on the first :meth:`Process.name` +call) and the other values are cached. Subsequent calls sharing the same +internal routine (read :proc:`/proc/pid/stat`, call ``sysctl()`` or whatever) +return the cached value. The code above can now be rewritten like this, and on +Linux it runs 2.4 times faster: + +.. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process() + >>> with p.oneshot(): + ... p.name() + ... p.cpu_times() + ... p.create_time() + ... p.ppid() + ... p.status() + ... p.terminal() + +Implementation +-------------- + +One great thing about psutil's design is its abstraction. It is divided into 3 +"layers". Layer 1 is represented by the main +`Process class `__ +(Python), which exposes the high-level API. Layer 2 is the +`OS-specific Python module `__, +which is a thin wrapper on top of the OS-specific +`C extension module `__ +(layer 3). + +Because the code was organized this way (modular), the refactoring was +reasonably smooth. I first refactored those C functions that collect multiple +pieces of info and grouped them into a single function (e.g. see +`BSD implementation `__). +Then I wrote a +`decorator `__ +that enables the cache only when requested (when entering the context manager), +and decorated the +`"grouped functions" `__ +with it. The caching mechanism is controlled by the :meth:`Process.oneshot` +context manager, which is the only thing exposed to the end user. Here's the +decorator: + +.. code-block:: python + + def memoize_when_activated(fun): + """A memoize decorator which is disabled by default. It can be + activated and deactivated on request. + """ + @functools.wraps(fun) + def wrapper(self): + if not wrapper.cache_activated: + return fun(self) + else: + try: + ret = cache[fun] + except KeyError: + ret = cache[fun] = fun(self) + return ret + + def cache_activate(): + """Activate cache.""" + wrapper.cache_activated = True + + def cache_deactivate(): + """Deactivate and clear cache.""" + wrapper.cache_activated = False + cache.clear() + + cache = {} + wrapper.cache_activated = False + wrapper.cache_activate = cache_activate + wrapper.cache_deactivate = cache_deactivate + return wrapper + +To measure the speedup I wrote a +`benchmark script `__ +(well, +`two `__ +actually), and kept tuning until I was sure the change actually made psutil +faster. The scripts report the speedup for calling all the "grouped" methods +together (best-case scenario). + +Linux: +2.56x speedup +--------------------- + +The Linux implementation is mostly Python, reading files in ``/proc``. These +files typically expose multiple pieces of info per process; +:proc:`/proc/pid/stat` and :proc:`/proc/pid/status` are the perfect example. We +aggregate them into three groups. See the relevant code +`here `__. + +Windows: from +1.9x to +6.5x speedup +------------------------------------ + +Windows is an interesting one. For a process owned by our user, we group only +:meth:`Process.num_threads`, :meth:`Process.num_ctx_switches` and +:meth:`Process.num_handles`, for a +1.9x speedup if we access those methods in +one shot. + +Windows is special though, because certain methods have a dual implementation +(:gh:`304`): a "fast method" is tried first, but if the process is owned by +another user it fails with :exc:`AccessDenied`. psutil then falls back to a +second, "slower" method (see `here +`__ +for example). + +It's slower because it +`iterates over all PIDs `__, +but unlike the "plain" Windows APIs it can still +`retrieve multiple pieces of information in one shot `__: +number of threads, :term:`context switches `, handles, CPU +times, create time, and I/O counters. + +That's why querying processes owned by other users results in an impressive ++6.5x speedup. + +macOS: +1.92x speedup +--------------------- + +On macOS we can get 2 groups of information. With +`sysctl() `__ +we get process parent PID, uids, gids, terminal, create time, name. With +`proc_info() `__ +we get CPU times (for PIDs owned by another user), memory metrics and ctx +switches. Not bad. + +BSD: +2.18x speedup +------------------- + +On BSD we gather tons of process info just by calling ``sysctl()`` (see +`implementation `__): +process name, ppid, status, uids, gids, IO counters, CPU and create times, +terminal and ctx switches. + +SunOS: +1.37x speedup +--------------------- + +SunOS is like Linux (it reads files in ``/proc``), but the code is in C. Here +too, we group different metrics together (see +`here `__ +and +`here `__). + +Discussion +---------- + +* `Reddit `__ diff --git a/docs/blog/2016/netbsd-support.rst b/docs/blog/2016/netbsd-support.rst new file mode 100644 index 0000000000..be8178d50c --- /dev/null +++ b/docs/blog/2016/netbsd-support.rst @@ -0,0 +1,50 @@ +.. post:: 2016-02-25 + :tags: bsd, new-platform, community, release + :author: Giampaolo Rodola + :exclude: + + Completing the BSD trio alongside FreeBSD and OpenBSD + +NetBSD support +============== + +Roughly two months have passed since I last +:doc:`announced ` that psutil added support for +OpenBSD. Today I'm happy to announce that it's the turn of NetBSD! This was +contributed by `Thomas Klausner `_, +`Ryo Onodera `_ and myself in :pr:`557`. + +Differences with FreeBSD (and OpenBSD) +-------------------------------------- + +The NetBSD implementation has limitations similar to the ones I encountered +with OpenBSD. Again, FreeBSD remains the BSD variant with the best support in +terms of kernel functionality. + +* :meth:`Process.memory_maps` is not implemented. The kernel provides the + necessary pieces, but I haven't done this yet (hopefully later). +* :meth:`Process.num_ctx_switches`'s :field:`involuntary` field is always 0. + The ``kinfo_proc()`` syscall provides this info, but it's always set to 0. +* :meth:`Process.cpu_affinity` (get and set) is not supported. +* :func:`psutil.cpu_count` ``(logical=False)`` always returns ``None``. + +As for the rest: it is all there. All memory, disk, network and process APIs +are fully supported and functioning. + +Other enhancements available in this psutil release +--------------------------------------------------- + +Besides NetBSD support, this release has a couple of interesting enhancements: + +* :gh:`708`: [Linux] :func:`psutil.net_connections` and + :meth:`Process.connections` can be up to 3x faster when there are many + connections. +* :gh:`718`: :func:`psutil.process_iter` is now thread safe. + +You can read the rest in the :ref:`changelog <341>`, as usual. + +Discussion +---------- + +* `Reddit `_ +* `Hacker News `_ diff --git a/docs/blog/2016/real-process-memory-in-python.rst b/docs/blog/2016/real-process-memory-in-python.rst new file mode 100644 index 0000000000..1ec8ac37b4 --- /dev/null +++ b/docs/blog/2016/real-process-memory-in-python.rst @@ -0,0 +1,210 @@ +.. post:: 2016-02-17 + :tags: memory, new-api, compatibility, release, community, linux + :author: Giampaolo Rodola + :exclude: + + psutil 4.0.0 exposes :term:`USS`, the memory that would + actually be freed on process exit + :meth:`Process.environ` + +Real process memory and environ in Python +========================================= + +psutil 4.0.0 is out, with some interesting news about process memory metrics. +I'll get straight to the point and describe what's new. + +"Real" process memory info +-------------------------- + +Determining how much memory a process **really** uses is not an easy matter +(see `this `__ and +`this `__). +:term:`RSS` (Resident Set Size), which most people rely on, is misleading +because it includes both memory unique to the process and memory shared with +others. What's more interesting for profiling is the memory that would be freed +if the process were terminated **right now**. In the Linux world this is called +:term:`USS` (Unique Set Size), the major feature introduced in psutil 4.0.0 +(not only for Linux but also for Windows and macOS). + +USS memory +---------- + +The :term:`USS` (Unique Set Size) is the memory unique to a process, that would +be freed if the process were terminated right now. On Linux it can be +determined by parsing the "private" blocks in :proc:`/proc/pid/smaps`. The +Firefox team pushed this further and got it working on +`macOS and Windows `__ +too. + +.. code-block:: pycon + + >>> psutil.Process().memory_full_info() + pfullmem(rss=101990, vms=521888, shared=38804, text=28200, lib=0, data=59672, dirty=0, uss=81623, pss=91788, swap=0) + +PSS and swap +------------ + +On Linux there are two additional metrics that can also be determined via +:proc:`/proc/pid/smaps`: :term:`PSS` and :term:`swap `. + +:field:`pss`, aka "Proportional Set Size", represents the amount of memory +shared with other processes, accounted so that the amount is divided evenly +between the processes that share it. I.e. if a process has 10 MBs all to itself +(:term:`USS`) and 10 MBs shared with another process, its :term:`PSS` will be +15 MBs. + +:field:`swap` is simply the amount of memory that has been +:term:`swapped out ` to disk. With :meth:`Process.memory_full_info` +it is possible to implement a tool like :src:`scripts/procsmem.py`, similar to +`smem `__ on Linux, which provides a list of +processes sorted by :field:`uss`. It's interesting to see how :field:`rss` +differs from :field:`uss`: + +.. code-block:: none + + ~/svn/psutil$ ./scripts/procsmem.py + PID User Cmdline USS PSS Swap RSS + ============================================================================== + ... + 3986 giampao /usr/bin/python3 /usr/bin/indi 15.3M 16.6M 0B 25.6M + 3906 giampao /usr/lib/ibus/ibus-ui-gtk3 17.6M 18.1M 0B 26.7M + 3991 giampao python /usr/bin/hp-systray -x 19.0M 23.3M 0B 40.7M + 3830 giampao /usr/bin/ibus-daemon --daemoni 19.0M 19.0M 0B 21.4M + 20529 giampao /opt/sublime_text/plugin_host 19.9M 20.1M 0B 22.0M + 3990 giampao nautilus -n 20.6M 29.9M 0B 50.2M + 3898 giampao /usr/lib/unity/unity-panel-ser 27.1M 27.9M 0B 37.7M + 4176 giampao /usr/lib/evolution/evolution-c 35.7M 36.2M 0B 41.5M + 20712 giampao /usr/bin/python -B /home/giamp 45.6M 45.9M 0B 49.4M + 3880 giampao /usr/lib/x86_64-linux-gnu/hud/ 51.6M 52.7M 0B 61.3M + 20513 giampao /opt/sublime_text/sublime_text 65.8M 73.0M 0B 87.9M + 3976 giampao compiz 115.0M 117.0M 0B 130.9M + 32486 giampao skype 145.1M 147.5M 0B 149.6M + +Implementation +-------------- + +To get these values (:field:`uss`, :field:`pss` and :field:`swap`) we need to +walk the whole process address space. This usually requires higher privileges +and is considerably slower than :meth:`Process.memory_info`, which is probably +why tools like ``ps`` and ``top`` show :term:`RSS`/:term:`VMS` instead of +:term:`USS`. A big thanks goes to the Mozilla team for figuring this out on +Windows and macOS, and to `Eric Rahm `_ who put +the psutil PRs together (see :pr:`744`, :pr:`745` and :pr:`746`). If you don't +use Python and want to port the code to another language, here are the +interesting parts: + +* `Linux `__ +* `macOS `__ +* `Windows `__ + +Memory type percent +------------------- + +After reorganizing the process memory APIs (:pr:`744`), I added a new +``memtype`` parameter to :meth:`Process.memory_percent`. You can now compare a +specific memory type (not only :term:`RSS`) against the total physical memory. +E.g. + +.. code-block:: pycon + + >>> psutil.Process().memory_percent(memtype='pss') + 0.06877466326787016 + +Process environ +--------------- + +The second biggest improvement in psutil 4.0.0 is the ability to read a +process's environment variables. This opens up interesting possibilities for +process recognition and monitoring. For instance, you can start a process with +a custom environment variable, then iterate over all processes to find the one +of interest: + +.. code-block:: python + + import psutil + for p in psutil.process_iter(): + try: + env = p.environ() + except psutil.Error: + pass + else: + if 'MYAPP' in env: + ... + +Process environ was a long-standing issue (:gh:`52`, from 2009) that I gave up +on because the Windows implementation only worked for the current process. +`Frank Benkstein `_ solved that (:pr:`747`), and +it now works on Linux, Windows and macOS for all processes (you may still hit +:exc:`AccessDenied` for processes owned by another user): + +.. code-block:: pycon + + >>> import psutil + >>> from pprint import pprint as pp + >>> pp(psutil.Process().environ()) + {... + 'CLUTTER_IM_MODULE': 'xim', + 'COLORTERM': 'gnome-terminal', + 'COMPIZ_BIN_PATH': '/usr/bin/', + 'HOME': '/home/giampaolo', + 'PWD': '/home/giampaolo/svn/psutil', + } + >>> + +Note that the resulting dict usually doesn't reflect changes made after the +process started (e.g. ``os.environ['MYAPP'] = '1'``). Again, for anyone porting +this to other languages, here are the interesting parts: + +* `Linux `_ +* `macOS `_ +* Windows: :pr:`747` + +Extended disk IO stats +---------------------- + +:func:`psutil.disk_io_counters` now reports additional metrics on Linux and +FreeBSD: + +* :field:`busy_time`: the time spent doing actual I/Os (in milliseconds). +* :field:`read_merged_count` and :field:`write_merged_count` (Linux only): the + number of merged reads and writes (see the + `iostats `_ doc). + +These give a better picture of actual disk utilization (:gh:`756`), similar to +the ``iostat`` command on Linux. + +OS constants +------------ + +Given the growing number of platform-specific metrics, I added a set of +constants to tell which platform you're on: :data:`psutil.LINUX`, +:data:`psutil.WINDOWS`, etc. + +Other fixes +----------- + +The complete list of changes is available in the :ref:`changelog <400>`. + +Porting code +------------ + +Since 4.0.0 is a major version, I took the chance to (lightly) change / break +some APIs. + +* :meth:`Process.memory_info` no longer returns just an (:field:`rss`, + :field:`vms`) namedtuple. It returns a variable-length namedtuple that varies + by platform (:field:`rss` and :field:`vms` are always present, even on + Windows). Essentially the same result as the old + ``Process.memory_info_ex()``. This shouldn't break your code unless you were + doing ``rss, vms = p.memory_info()``. +* ``Process.memory_info_ex()`` is deprecated. It still works as an alias for + :meth:`Process.memory_info`, issuing a :exc:`DeprecationWarning`. +* :func:`psutil.disk_io_counters` on NetBSD and OpenBSD no longer returns + :field:`write_count` and :field:`read_count` because the kernel doesn't + provide them (we were returning the busy time instead). Should be a small + issue given NetBSD and OpenBSD support is very recent. + +Discussion +---------- + +* `Reddit `_ +* `Hacker News `_ diff --git a/docs/blog/2016/windows-services.rst b/docs/blog/2016/windows-services.rst new file mode 100644 index 0000000000..1770378ea6 --- /dev/null +++ b/docs/blog/2016/windows-services.rst @@ -0,0 +1,107 @@ +.. post:: 2016-05-15 + :tags: windows, new-api, release + :author: Giampaolo Rodola + :exclude: + + psutil 4.2.0 introduces :func:`win_service_iter` and :func:`win_service_get` + +Windows services support +======================== + +New psutil 4.2.0 is out. The highlight of this release is the support for +Windows services (executables that run at system startup, similar to UNIX init +scripts): + +.. code-block:: pycon + + >>> import psutil + >>> list(psutil.win_service_iter()) + [, + , + , + , + ...] + >>> s = psutil.win_service_get('alg') + >>> s.as_dict() + {'binpath': 'C:\\Windows\\System32\\alg.exe', + 'description': 'Provides support for 3rd party protocol plug-ins for Internet Connection Sharing', + 'display_name': 'Application Layer Gateway Service', + 'name': 'alg', + 'pid': None, + 'start_type': 'manual', + 'status': 'stopped', + 'username': 'NT AUTHORITY\\LocalService'} + +I decided to do this mainly because I find pywin32 APIs too low levelish. +Having something like this in psutil can be useful to discover and monitor +services more easily. The code was implemented in :pr:`803`. The API for +querying a service is similar to :class:`psutil.Process`. You can get a +reference to a service object by using its name (which is unique for every +service) and then use methods like :meth:`WindowsService.name` and +:meth:`WindowsService.status`: + +.. code-block:: pycon + + >>> s = psutil.win_service_get('alg') + >>> s.name() + 'alg' + >>> s.status() + 'stopped' + +Initially I thought about providing a full set of APIs to handle all aspects of +service management, including ``start()``, ``stop()``, ``restart()``, +``install()``, ``uninstall()`` and ``modify()``. However, I soon realized I +would have ended up reimplementing what pywin32 already provides, at the cost +of overcrowding the psutil API (see my reasoning +`here `__). +I think psutil really focuses on monitoring, not on installing and modifying +system components, especially something as critical as a Windows service. + +Considerations about Windows services +------------------------------------- + +Typically, a Windows service is an executable (.exe) that runs at system +startup and continues running in the background. It is roughly the equivalent +of a UNIX init script. All services are controlled by a "manager", which keeps +track of their status and metadata (e.g. description, startup type). It is +interesting to note that since (most) services are bound to an executable (and +hence a process) you can reference them via their process PID: + +.. code-block:: pycon + + >>> s = psutil.win_service_get('sshd') + >>> s + + >>> s.pid() + 1865 + >>> p = psutil.Process(1865) + >>> p + + >>> p.exe() + 'C:\CygWin\bin\sshd' + +Other improvements +------------------ + +psutil 4.2.0 comes with 2 other enhancements for Linux: + +* :func:`psutil.virtual_memory` returns a new :field:`shared` memory field. + This is the same value reported by ``free`` cmdline utility. +* I changed how ``/proc`` was parsed. Instead of reading + :proc:`/proc/pid/status` line by line I used a regular expression. Here's the + speedups: + + * :meth:`Process.ppid` ~20% faster. + + * :meth:`Process.status` ~28% faster. + + * :meth:`Process.name` ~25% faster. + + * :meth:`Process.num_threads` ~20% faster (on Python 3 only; on Python 2 it's + a bit slower; I suppose :mod:`re` module received some improvements). + +Discussion +---------- + +* `Reddit `__ +* `Hacker News `__ diff --git a/docs/blog/2017/aix-support.rst b/docs/blog/2017/aix-support.rst new file mode 100644 index 0000000000..a7ac4646aa --- /dev/null +++ b/docs/blog/2017/aix-support.rst @@ -0,0 +1,57 @@ +.. post:: 2017-10-12 + :tags: new-platform, community, personal, release + :author: Giampaolo Rodola + :exclude: + + psutil 5.4.0 gets an IBM port, contributed by Arnon Yaari + +AIX support +=========== + +After a long wait psutil finally supports a new exotic platform: AIX! + +Honestly I'm not sure how many AIX Python users are out there (probably not +many), but here it is. + +For this we have to thank `Arnon Yaari `__, who +started working on the port a couple of years ago (:gh:`605`). I was skeptical +at first, because AIX is the only platform I can't virtualize and test on my +laptop, so that made me a bit nervous. Arnon did a great job. The final +:pr:`1123` is huge: it required a considerable amount of work on his part, and +a review of more than 140 messages exchanged between us over about a month, +during which I was travelling through China. + +The end result is very good: almost all original unit tests pass, and code +quality is awesome, which (I must say) is fairly unusual for an external +contribution like this. Kudos to you, Arnon! ;-) + +Other changes +------------- + +Besides AIX support, release 5.4.0 also includes a couple of important bug +fixes for :func:`psutil.sensors_temperatures` and :func:`psutil.sensors_fans` +on Linux, and a fix for a bug on macOS that could cause a segmentation fault +when using :meth:`Process.open_files`. The complete list of bug fixes is in the +:ref:`changelog <540>`. + +The future +---------- + +Looking ahead at other exotic, still-unsupported platforms, two contributions +are worth mentioning: a (still incomplete) PR for Cygwin which looks promising +(:pr:`998`), and Mingw32 compiler support on Windows (:pr:`845`). + +psutil is gradually reaching a point where adding new features is becoming +rarer, so it's a good moment to welcome new platforms while the API is mature +and stable. + +Future work along these lines could also include Android and (hopefully) iOS +support. Now *that* would be really awesome to have. + +Stay tuned. + +Discussion +---------- + +* `Reddit `__ +* `Blogspot `__ diff --git a/docs/blog/2017/process-iter-attrs.rst b/docs/blog/2017/process-iter-attrs.rst new file mode 100644 index 0000000000..65e85034ae --- /dev/null +++ b/docs/blog/2017/process-iter-attrs.rst @@ -0,0 +1,90 @@ +.. post:: 2017-09-03 + :tags: api-design, release, performance, new-api + :author: Giampaolo Rodola + :exclude: + + 5.3.0 adds ``attrs`` and ``ad_value`` parameters to + :func:`~psutil.process_iter`, letting you pre-fetch attributes in one shot + +Improved process_iter() +======================= + +This is part of the psutil 5.3.0 release (see the :ref:`changelog <530>` for +the full list of changes). + +The old pattern +--------------- + +Iterating over processes and collecting attributes requires more boilerplate +than it should. A process returned by :func:`process_iter` may disappear before +you access it, or require elevated privileges, so every lookup has to be +guarded with a ``try / except``: + +.. code-block:: pycon + + >>> import psutil + >>> for proc in psutil.process_iter(): + ... try: + ... pinfo = proc.as_dict(attrs=['pid', 'name']) + ... except (psutil.NoSuchProcess, psutil.AccessDenied): + ... pass + ... else: + ... print(pinfo) + ... + {'pid': 1, 'name': 'systemd'} + {'pid': 2, 'name': 'kthreadd'} + {'pid': 3, 'name': 'ksoftirqd/0'} + +This is not decorative. It's necessary to avoid the race condition. + +The new pattern +--------------- + +5.3.0 adds ``attrs`` and ``ad_value`` parameters to +:func:`psutil.process_iter`. With these, the loop body becomes: + +.. code-block:: pycon + + >>> import psutil + >>> for proc in psutil.process_iter(attrs=['pid', 'name']): + ... print(proc.info) + ... + {'pid': 1, 'name': 'systemd'} + {'pid': 2, 'name': 'kthreadd'} + {'pid': 3, 'name': 'ksoftirqd/0'} + +Internally, :func:`process_iter` attach an ``info`` dict to the +:class:`Process` instance. The attributes are pre-fetched in one shot. +Processes that disappear during iteration are silently skipped, and attributes +that would raise :exc:`AccessDenied` gets assigned ``ad_value`` , which +defaults to ``None``: + +.. code-block:: python + + for p in psutil.process_iter(['name', 'username'], ad_value="N/A"): + print(p.name(), p.username()) + +Performance +----------- + +Beyond the syntactic win, the new syntax is also faster than calling individual +methods in a loop. ``process_iter(attrs=[...])`` is equivalent to using +:meth:`Process.oneshot` on each process (see +:doc:`One shot, twice as fast ` for how that +works): attributes that share a syscall or a ``/proc`` file are fetched +together instead of re-read on every method call, which is a lot faster. + +Comprehensions +-------------- + +With the exception boilerplate out of the way, comprehensions finally work +cleanly. E.g. getting processes owned by the current user can be written as: + +.. code-block:: pycon + + >>> import getpass + >>> from pprint import pprint as pp + >>> pp([(p.pid, p.info['name']) for p in psutil.process_iter(attrs=['name', 'username']) if p.info['username'] == getpass.getuser()]) + [(16832, 'bash'), + (19772, 'ssh'), + (20492, 'python')] diff --git a/docs/blog/2017/sensors-support.rst b/docs/blog/2017/sensors-support.rst new file mode 100644 index 0000000000..34b071e0b7 --- /dev/null +++ b/docs/blog/2017/sensors-support.rst @@ -0,0 +1,105 @@ +.. post:: 2017-02-01 + :tags: new-api, release, linux + :author: Giampaolo Rodola + :exclude: + + psutil 5.1.0 introduces :func:`sensors_temperatures`, + :func:`sensors_battery` and :func:`cpu_freq` + +Sensors: temperatures, battery, CPU frequency +============================================= + +psutil 5.1.0 is out. This release introduces new APIs to retrieve hardware +temperatures, battery status, and CPU frequency information. + +Temperatures +------------ + +You can now retrieve hardware temperatures (:pr:`962`). This is currently +available on Linux only. + +* On Windows it's hard to do in a hardware-agnostic way. I ran into 3 WMI-based + approaches, none of which worked with my hardware, so I gave up. +* On macOS it seems relatively easy, but my virtualized macOS box doesn't + support sensors, so I gave up for lack of hardware. If someone wants to give + it a try, be my guest (:gh:`371`). + +.. code-block:: pycon + + >>> import psutil + >>> psutil.sensors_temperatures() + {'acpitz': [shwtemp(label='', current=47.0, high=103.0, critical=103.0)], + 'asus': [shwtemp(label='', current=47.0, high=None, critical=None)], + 'coretemp': [shwtemp(label='Physical id 0', current=52.0, high=100.0, critical=100.0), + shwtemp(label='Core 0', current=45.0, high=100.0, critical=100.0), + shwtemp(label='Core 1', current=52.0, high=100.0, critical=100.0), + shwtemp(label='Core 2', current=45.0, high=100.0, critical=100.0), + shwtemp(label='Core 3', current=47.0, high=100.0, critical=100.0)]} + +Battery status +-------------- + +Battery status information is now available on Linux, Windows and FreeBSD +(:pr:`963`). + +.. code-block:: pycon + + >>> import psutil + >>> + >>> def secs2hours(secs): + ... mm, ss = divmod(secs, 60) + ... hh, mm = divmod(mm, 60) + ... return "%d:%02d:%02d" % (hh, mm, ss) + ... + >>> battery = psutil.sensors_battery() + >>> battery + sbattery(percent=93, secsleft=16628, power_plugged=False) + >>> print("charge = %s%%, time left = %s" % (battery.percent, secs2hours(battery.secsleft))) + charge = 93%, time left = 4:37:08 + +CPU frequency +------------- + +Available on Linux, Windows and macOS (:pr:`952`). Only Linux reports the +real-time value (always changing); other platforms return the nominal "fixed" +value. + +.. code-block:: pycon + + >>> import psutil + >>> psutil.cpu_freq() + scpufreq(current=931.42925, min=800.0, max=3500.0) + >>> psutil.cpu_freq(percpu=True) + [scpufreq(current=2394.945, min=800.0, max=3500.0), + scpufreq(current=2236.812, min=800.0, max=3500.0), + scpufreq(current=1703.609, min=800.0, max=3500.0), + scpufreq(current=1754.289, min=800.0, max=3500.0)] + +What CPU a process is on +------------------------ + +Tells you which CPU a process is currently running on, somewhat related to +:meth:`Process.cpu_affinity` (:pr:`954`). It's interesting for visualizing how +the OS scheduler keeps evenly reassigning processes across CPUs (see the +:src:`scripts/cpu_distribution.py` script). + +CPU affinity +------------ + +A new shorthand is available to set affinity against all eligible CPUs: + +.. code-block:: python + + Process().cpu_affinity([]) + +This was added because on Linux (:gh:`956`) it is not always possible to set +affinity against all CPUs directly. It is equivalent to: + +.. code-block:: python + + psutil.Process().cpu_affinity(list(range(psutil.cpu_count()))) + +Other bug fixes +--------------- + +See the full list in the :ref:`changelog <510>`. diff --git a/docs/blog/2017/unicode-internals.rst b/docs/blog/2017/unicode-internals.rst new file mode 100644 index 0000000000..15dd92ec17 --- /dev/null +++ b/docs/blog/2017/unicode-internals.rst @@ -0,0 +1,107 @@ +.. post:: 2017-09-03 + :tags: api-design, compatibility, python-core + :author: Giampaolo Rodola + :exclude: + + How psutil 5.3.0 got non-ASCII strings right on both Python 2 and + Python 3. + +Fixing Unicode across Python 2 and 3 +==================================== + +This one took a while. Adding proper Unicode support to psutil took four months +of auditing, design decisions, and rewriting nearly every API that returned a +string. The full journey is documented in :gh:`1040`, and what follows is a +summary. + +This can serve as a case study for any Python library with a C extension that +needs to support both Python 2 *and* Python 3, as it will encounter the exact +same set of problems. + +What was broken +--------------- + +psutil has different APIs returning a string, many of which misbehaved when it +came to unicode. There were three distinctive problems (:gh:`1040`). Each API +could: + +* **A**: raise a decoding error for non-ASCII strings (Python 3). +* **B**: return ``unicode`` instead of ``str`` (Python 2). +* **C**: return incorrect / invalid encoded data for non-ASCII strings (both). + +:meth:`Process.memory_maps` hit all three on various OSes. +:func:`disk_partitions` raised decoding errors on every UNIX except Linux. +Windows service methods leaked ``unicode`` into Python 2 return values. The C +extension had accumulated years of ad-hoc encode/decode decisions, with no +single rule covering all of them. + +It was a mess. + +Filesystem or locale encoding? +------------------------------ + +First problem was that the C extension was using 2 approaches when it came to +decoding and returning a string: :c:func:`PyUnicode_DecodeFSDefault` +(filesystem encoding) for path-like APIs, and :c:func:`PyUnicode_DecodeLocale` +(user locale) for non-path strings like :meth:`Process.username`. + +It appeared clear that I had to use :c:func:`PyUnicode_DecodeFSDefault` for all +filesystem-related APIs like :meth:`Process.exe` and +:meth:`Process.open_files`. + +It was less clear, though, when to use :c:func:`PyUnicode_DecodeLocale`. + +After some back and forth, I decided to use a single encoding for all APIs: the +**filesystem encoding** (:c:func:`PyUnicode_DecodeFSDefault`). This makes the +encoding choice an implementation detail of psutil, not something the user has +to care about. + +Error handling +-------------- + +Second question was what to do in case the string cannot be correctly decoded +(because invalid, corrupted or whatever). On Python 3 + UNIX the natural choice +was ``'surrogateescape'``, which is also the default for +:c:func:`PyUnicode_DecodeFSDefault`. On Windows the default is +``'surrogatepass'`` (Python 3.6) or ``'replace'`` as per +`PEP 529 `__. + +And here come the troubles: Python 2 is different. To correctly handle all +kinds of strings on Python 2 we should return ``unicode`` instead of ``str``, +but I didn't want to do that, nor have APIs which return two different types +depending on the circumstance. + +Since unicode support is already broken in Python 2 and its stdlib (see +:bpo:`18695`), I was happy to always return ``str``, use ``'replace'`` as the +error handler, and simply consider unicode support in psutil + Python 2 broken. + +Final behavior +-------------- + +Starting from 5.3.0, psutil behaves consistently across all APIs that return a +string. The rules are intentionally simple, even if the underlying +implementation is not. + +The notes below apply to *any* method returning a string such as +:meth:`Process.exe` or :meth:`Process.cwd`, including non-filesystem-related +methods such as :meth:`Process.username`: + +* all strings are encoded using the OS filesystem encoding + (:c:func:`PyUnicode_DecodeFSDefault`), which varies depending on the platform + you're on (e.g. ``'UTF-8'`` on Linux, ``'mbcs'`` on Windows). +* no API call is supposed to crash with :exc:`UnicodeDecodeError`. +* in case of badly encoded data returned by the OS, the following error + handlers are used to replace the bad characters in the string: + + - Python 2: ``'replace'``. + - Python 3: ``'surrogateescape'`` on POSIX, ``'replace'`` on Windows. + +* on Python 2 all APIs return bytes (``str`` type), never ``unicode``. +* on Python 2 you can go back to unicode by doing: + + .. code-block:: pycon + + >>> unicode(proc.exe(), sys.getdefaultencoding(), errors="replace") + +The full journey was implemented in :pr:`1052`, and shipped in 5.3.0 (see the +:ref:`changelog <530>`). diff --git a/docs/blog/2019/announcing-560.rst b/docs/blog/2019/announcing-560.rst new file mode 100644 index 0000000000..57fcb2afa3 --- /dev/null +++ b/docs/blog/2019/announcing-560.rst @@ -0,0 +1,74 @@ +.. post:: 2019-03-05 + :tags: windows, macos, new-api, compatibility, release + :author: Giampaolo Rodola + :exclude: + + 5.6.0 adds :meth:`~psutil.Process.parents`, brings several Windows improvements, and removes :meth:`~psutil.Process.memory_maps` on macOS + +Announcing psutil 5.6.0 +======================= + +psutil 5.6.0 is out. Highlights: a new :meth:`Process.parents` method, several +important Windows improvements, and the removal of :meth:`Process.memory_maps` +on macOS. + +Process parents() +----------------- + +The new method returns the parents of a process as a list of :class:`Process` +instances. If no parents are known, an empty list is returned. + +.. code-block:: pycon + + >>> import psutil + >>> p = psutil.Process(5312) + >>> p.parents() + [psutil.Process(pid=4699, name='bash', started='09:06:44'), + psutil.Process(pid=4689, name='gnome-terminal-server', started='09:06:44'), + psutil.Process(pid=1, name='systemd', started='05:56:55')] + +Nothing fundamentally new here, since this is a convenience wrapper around +:meth:`Process.parent`, but it's still nice to have it built in. It pairs well +with :meth:`Process.children` when working with process trees. The idea was +proposed by Ghislain Le Meur. + +Windows +------- + +Certain Windows APIs that need to be dynamically loaded from DLLs are now +loaded only once at startup, instead of on every function call. This makes some +operations **50% to 100% faster**; see benchmarks in :pr:`1422`. + +:meth:`Process.suspend` and :meth:`Process.resume` previously iterated over all +process threads via ``CreateToolhelp32Snapshot()``, which was unorthodox and +broke when the process had been suspended by Process Hacker. They now call the +undocumented ``NtSuspendProcess()`` / ``NtResumeProcess()`` NT APIs, same as +Process Hacker and Sysinternals tools. Discussed in :gh:`1379`, implemented in +:pr:`1435`. + +``SE DEBUG`` is a privilege bit set on the Python process at startup so psutil +can query processes owned by other users (Administrator, Local System), meaning +fewer :exc:`AccessDenied` exceptions for low-PID processes. The code setting it +had presumably been broken for years and is now finally fixed in :pr:`1429`. + +Removal of Process.memory_maps() on macOS +----------------------------------------- + +:meth:`Process.memory_maps` is gone on macOS (:gh:`1291`). The underlying Apple +API would randomly raise ``EINVAL`` or segfault the host process, and no amount +of reverse-engineering produced a safe fix. So I removed it. This is covered in +a :doc:`separate post `. + +Improved exceptions +------------------- + +One problem that affected psutil maintenance over the years was receiving bug +reports whose tracebacks did not indicate which syscall had actually failed. +This was especially painful on Windows, where a single routine may invoke +multiple Windows APIs. Now the :exc:`OSError` (or ``WindowsError``) exception +includes the syscall from which the error originated. See :pr:`1428`. + +Other changes +------------- + +See the :ref:`changelog <560>`. diff --git a/docs/blog/2019/macos-memory-maps.rst b/docs/blog/2019/macos-memory-maps.rst new file mode 100644 index 0000000000..a7f50489d7 --- /dev/null +++ b/docs/blog/2019/macos-memory-maps.rst @@ -0,0 +1,58 @@ +.. post:: 2019-03-05 + :tags: macos, api-design, compatibility, release + :author: Giampaolo Rodola + :exclude: + + A segfault I couldn't fix, and the API removal that followed + +Removing Process.memory_maps() on macOS +======================================= + +This is part of the psutil 5.6.0 release (see the full +:doc:`release notes `). + +As of 5.6.0, :meth:`Process.memory_maps` is no longer defined on macOS. + +The bug +------- + +:gh:`1291`: on macOS, :meth:`Process.memory_maps` would either raise +``OSError: [Errno 22] Invalid argument`` or segfault the whole Python process! +Both triggered from code as simple as ``psutil.Process().as_dict()``, since +:meth:`~Process.as_dict` iterates every attribute, and +:meth:`~Process.memory_maps` is one of them. + +The root cause was inside Apple's undocumented ``proc_regionfilename()`` +syscall. On some memory regions it returns ``EINVAL``. On others it takes the +process down. Which regions? Nobody figured out. Arnon Yaari +(:user:`wiggin15`) did most of the +investigation: he wrote a `standalone C reproducer +`__ +and walked me through what he'd tried. + +In :pr:`1436` I attempted a fix by reverse-engineering ``vmmap(1)`` but it +didn't work. The fundamental problem is that ``vmmap`` is closed source and +``proc_regionfilename`` is undocumented. Neither my virtualized macOS (10.11.6) +nor Travis CI (10.12.1) could reproduce the bug, which reproduced reliably only +on 10.14.3. + +Why remove outright +------------------- + +While removing the C code I noticed that the macOS unit test had been disabled +long ago, presumably by me after recurring flaky Travis runs. Meaning that the +method had been broken on some macOS versions far longer than the 2018 bug +report suggested. + +Deprecating for a cycle didn't help either: raising :exc:`AccessDenied` breaks +code that relied on a successful return, returning an empty list does the same +silently, and leaving the method in place doesn't stop the segfault. Basically +there was no sane solution, so since 5.6 is a major version I decided to just +remove :meth:`Process.memory_maps` for good. + +On macOS it never supported other processes anyway. Calling it on any PID other +than the current one (or its children) raised :exc:`AccessDenied`, even as +root. + +If someone finds a Mach API path that works, the method can return. Nobody has +found one so far. diff --git a/docs/blog/2025/drop-py27.rst b/docs/blog/2025/drop-py27.rst new file mode 100644 index 0000000000..225549910c --- /dev/null +++ b/docs/blog/2025/drop-py27.rst @@ -0,0 +1,114 @@ +.. post:: 2025-02-13 + :tags: compatibility, release, featured + :author: Giampaolo Rodola + :exclude: + + Downloads fell from ~8% to 0.36% in three years; psutil + 7.0.0 ends support + +Letting go of Python 2.7 +======================== + +About dropping Python 2.7 support in psutil, 3 years ago I stated (:gh:`2014`): + + Not a chance, for many years to come. [Python 2.7] currently + represents 7-10% of total downloads, meaning around 70k / 100k + downloads per day. + +Only 3 years later, and to my surprise, +**downloads for Python 2.7 dropped to 0.36%**! As such, as of psutil 7.0.0, I +finally decided to drop support for Python 2.7! + +The numbers +----------- + +These are downloads per month: + +.. code-block:: none + + $ pypinfo --percent psutil pyversion + Served from cache: False + Data processed: 4.65 GiB + Data billed: 4.65 GiB + Estimated cost: $0.03 + + | python_version | percent | download_count | + | -------------- | ------- | -------------- | + | 3.10 | 23.84% | 26,354,506 | + | 3.8 | 18.87% | 20,862,015 | + | 3.7 | 17.38% | 19,217,960 | + | 3.9 | 17.00% | 18,798,843 | + | 3.11 | 13.63% | 15,066,706 | + | 3.12 | 7.01% | 7,754,751 | + | 3.13 | 1.15% | 1,267,008 | + | 3.6 | 0.73% | 803,189 | + | 2.7 | 0.36% | 402,111 | + | 3.5 | 0.03% | 28,656 | + | Total | | 110,555,745 | + +According to `pypistats.org `__ Python 2.7 +downloads represent 0.28% of the total, around 15,000 downloads per day. + +The pain +-------- + +Keeping 2.7 alive had become increasingly difficult, but still possible: tests +ran via +`old PyPI backports `__ +and a +`tweaked GitHub Actions workflow `__ +on Linux and macOS, plus a separate third-party service (Appveyor) for Windows. +But the workarounds in the source kept piling up: + +- A Python compatibility layer + (`psutil/\_compat.py `__) + plus ``#if PY_MAJOR_VERSION <= 3`` branches in C, with constant + str-vs-unicode juggling on both sides. +- No f-strings, and no free use of :mod:`enum` for constants (which ended up + with a different API shape than on Python 3). +- An outdated ``pip`` and + `outdated deps `__. +- 4 extra CI jobs per commit (Linux, macOS, Windows 32-bit and 64-bit), making + the pipeline slower and flakier. +- 7 wheels specific to Python 2.7 to ship on every release: + +.. code-block:: none + + psutil-6.1.1-cp27-cp27m-macosx_10_9_x86_64.whl + psutil-6.1.1-cp27-none-win32.whl + psutil-6.1.1-cp27-none-win_amd64.whl + psutil-6.1.1-cp27-cp27m-manylinux2010_i686.whl + psutil-6.1.1-cp27-cp27m-manylinux2010_x86_64.whl + psutil-6.1.1-cp27-cp27mu-manylinux2010_i686.whl + psutil-6.1.1-cp27-cp27mu-manylinux2010_x86_64.whl + +The removal +----------- + +The removal was done in :pr:`2481`, which dropped around 1500 lines of code +(nice!). **It felt liberating**. In doing so, in the doc I still made the +promise that the 6.1.\* series will keep supporting Python 2.7 and will receive +**critical bug-fixes only** (no new features). It will be maintained in a +specific +`python2 branch `__. I +explicitly kept the +`setup.py `__ +script compatible with Python 2.7 in terms of syntax, so that, when the tarball +is fetched from PyPI, it will emit an informative error message on +``pip install psutil``. The user trying to install psutil on Python 2.7 will +see: + +.. code-block:: none + + $ pip2 install psutil + As of version 7.0.0 psutil no longer supports Python 2.7. + Latest version supporting Python 2.7 is psutil 6.1.X. + Install it with: "pip2 install psutil==6.1.*". + +Related +------- + +- 2017-06: :gh:`1053` (2.7 ticket) +- 2022-04: :pr:`2099` (Drop 2.6) +- 2023-04: :pr:`2246` (Drop 3.4 & 3.5) +- 2024-12: :pr:`2481` (Drop 2.7) diff --git a/docs/blog/2025/heap-introspection-apis.rst b/docs/blog/2025/heap-introspection-apis.rst new file mode 100644 index 0000000000..9a9736cc86 --- /dev/null +++ b/docs/blog/2025/heap-introspection-apis.rst @@ -0,0 +1,212 @@ +.. post:: 2025-12-23 + :tags: memory, c, new-api, featured + :author: Giampaolo Rodola + :exclude: + + psutil 7.2.0 peeks into the native allocator, spotting leaks :term:`RSS` can't see + +Detecting memory leaks in C extensions with psutil and psleak +============================================================= + +Memory leaks in Python are usually straightforward to diagnose. Just look at +:term:`RSS`, track Python object counts, follow reference graphs, etc. But +leaks inside C extension modules are another story. Traditional memory metrics +such as :term:`RSS` and :term:`VMS` fail to reveal them because Python's memory +allocator +(`pymalloc `__) +sits above the platform's native :term:`heap`. If something in an extension +calls :manpage:`malloc(3)` without a corresponding :manpage:`free(3)`, that +memory often won't show up in :term:`RSS` / :term:`VMS`. You have a leak, and +you don't know. + +psutil 7.2.0 introduces two new APIs for **C :term:`heap` introspection**, +designed specifically to catch these kinds of native leaks. They give you a +window directly into the underlying platform allocator (e.g. glibc's malloc), +letting you track how much memory the C layer actually allocates. If your +:term:`RSS` is flat but your C :term:`heap` usage climbs, you now have a way to +see it. + +Why native heap introspection matters +------------------------------------- + +Many Python projects rely on C extensions: psutil, NumPy, pandas, PIL, lxml, +psycopg, PyTorch, custom in-house modules, etc. And even CPython itself, which +implements many of its standard library modules in C. If any of these +components mishandle memory at the C level, you get a leak that doesn't show up +in: + +- Python reference counts (:func:`sys.getrefcount`). +- :mod:`tracemalloc` module. +- Python's :mod:`gc` stats. +- :term:`RSS`, :term:`VMS` or :term:`USS` due to allocator caching, especially + for small objects. This can happen, for example, when you forget to + ``Py_DECREF`` a Python object. + +psutil's new functions let you query the allocator (e.g. glibc) directly, +returning low-level metrics from the platform's native heap. + +heap_info(): direct allocator statistics +---------------------------------------- + +:func:`psutil.heap_info` exposes the following metrics: + +- :field:`heap_used`: total number of bytes currently allocated via + ``malloc()`` (small allocations). +- :field:`mmap_used`: total number of bytes currently allocated via + :manpage:`mmap(2)` or via large ``malloc()`` allocations. +- :field:`heap_count`: (Windows only) number of private heaps created via + ``HeapCreate()``. + +Example: + +.. code-block:: pycon + + >>> import psutil + >>> psutil.heap_info() + pheap(heap_used=5177792, mmap_used=819200) + +Reference for what contributes to each field: + +.. list-table:: + :header-rows: 1 + + * - Platform + - Allocation type + - Field affected + * - UNIX / Windows + - small ``malloc()`` ≤128 KB without ``free()`` + - :field:`heap_used` + * - UNIX / Windows + - large ``malloc()`` >128 KB without ``free()``, or ``mmap()`` + without :manpage:`munmap(2)` (UNIX) + - :field:`mmap_used` + * - Windows + - ``HeapAlloc()`` without ``HeapFree()`` + - :field:`heap_used` + * - Windows + - ``VirtualAlloc()`` without ``VirtualFree()`` + - :field:`mmap_used` + * - Windows + - ``HeapCreate()`` without ``HeapDestroy()`` + - :field:`heap_count` + +heap_trim(): returning unused heap memory +----------------------------------------- + +:func:`psutil.heap_trim` provides a cross-platform way to request that the +underlying allocator free any unused memory it's holding in the heap (typically +small ``malloc()`` allocations). + +In practice, modern allocators rarely comply, so this is not a general-purpose +memory-reduction tool and won't meaningfully shrink :term:`RSS` in real +programs. Its primary value is in leak detection tools. Calling +:func:`psutil.heap_trim` before taking measurements helps reduce allocator +noise, giving you a cleaner baseline so that changes in :field:`heap_used` come +from the code you're testing, not from internal allocator caching or +fragmentation. + +Real-world use: finding a C extension leak +------------------------------------------ + +The workflow is simple: + +1. Take a baseline snapshot of the heap. +2. Call the C extension hundreds of times. +3. Take another snapshot. +4. Compare. + +.. code-block:: python + + import psutil + + psutil.heap_trim() # reduce noise + + before = psutil.heap_info() + for _ in range(200): + my_cext_function() + after = psutil.heap_info() + + print("delta heap_used =", after.heap_used - before.heap_used) + print("delta mmap_used =", after.mmap_used - before.mmap_used) + +If :field:`heap_used` or :field:`mmap_used` values increase consistently, +you've found a native leak. + +To reduce false positives, repeat the test multiple times, increasing the +number of calls on each retry. This approach helps distinguish real leaks from +random noise or transient allocations. + +A new tool: psleak +------------------ + +The strategy described above is exactly what I implemented in a new PyPI +package, which I called `psleak `__. It +runs the target function repeatedly, trims the allocator before each run, and +tracks differences across retries. Memory that grows consistently after several +runs is flagged as a leak. + +A minimal test suite looks like this: + +.. code-block:: python + + from psleak import MemoryLeakTestCase + + class TestLeaks(MemoryLeakTestCase): + def test_fun(self): + self.execute(some_c_function) + +If the function leaks memory, the test will fail with a descriptive exception: + +.. code-block:: none + + psleak.MemoryLeakError: memory kept increasing after 10 runs + Run # 1: heap=+388160 | uss=+356352 | rss=+327680 | (calls= 200, avg/call=+1940) + Run # 2: heap=+584848 | uss=+614400 | rss=+491520 | (calls= 300, avg/call=+1949) + Run # 3: heap=+778320 | uss=+782336 | rss=+819200 | (calls= 400, avg/call=+1945) + Run # 4: heap=+970512 | uss=+1032192 | rss=+1146880 | (calls= 500, avg/call=+1941) + Run # 5: heap=+1169024 | uss=+1171456 | rss=+1146880 | (calls= 600, avg/call=+1948) + Run # 6: heap=+1357360 | uss=+1413120 | rss=+1310720 | (calls= 700, avg/call=+1939) + Run # 7: heap=+1552336 | uss=+1634304 | rss=+1638400 | (calls= 800, avg/call=+1940) + Run # 8: heap=+1752032 | uss=+1781760 | rss=+1802240 | (calls= 900, avg/call=+1946) + Run # 9: heap=+1945056 | uss=+2031616 | rss=+2129920 | (calls=1000, avg/call=+1945) + Run #10: heap=+2140624 | uss=+2179072 | rss=+2293760 | (calls=1100, avg/call=+1946) + +Psleak is now part of the psutil test suite. All psutil APIs are tested (see +`test_memleaks.py `__), +making it a de facto **regression-testing tool**. + +It's worth noting that without inspecting heap metrics, missing calls in the C +code such as ``Py_CLEAR`` and ``Py_DECREF`` often go unnoticed, because they +don't affect :term:`RSS`, :term:`VMS`, and :term:`USS`. I confirmed this by +commenting them out. Monitoring the :term:`heap` is therefore essential to +reliably detect memory leaks in Python C extensions. + +Under the hood +-------------- + +For those interested in seeing how I did this in terms of code: + +- `Linux `__: + uses glibc's :manpage:`mallinfo2(3)` to report ``uordblks`` (heap + allocations) and ``hblkhd`` (mmap-backed blocks). +- `Windows `__: + enumerates heaps and aggregates ``HeapAlloc`` / ``VirtualAlloc`` usage. +- `macOS `__: + uses malloc zone statistics. +- `BSD `__: + uses jemalloc's arena and stats interfaces. + +References +---------- + +- `psleak `__, the new memory leak + testing framework. +- :pr:`2692`, the implementation. +- :gh:`1275`, the original proposal from 8 years earlier. + +Discussion +---------- + +- `Reddit `__ +- `Hacker News `__ +- `Medium `__ diff --git a/docs/blog/2025/speedup-pytest-startup.rst b/docs/blog/2025/speedup-pytest-startup.rst new file mode 100644 index 0000000000..1b6b639dfe --- /dev/null +++ b/docs/blog/2025/speedup-pytest-startup.rst @@ -0,0 +1,211 @@ +.. post:: 2025-04-04 + :tags: tests, performance + :author: Giampaolo Rodola + :exclude: + + Trimming heavy imports and plugins: 0.42s → 0.30s (~28%) + +Speeding up pytest startup +========================== + +Preface: the migration to pytest +-------------------------------- + +Last year, after 17 years of :mod:`unittest`, I started adopting pytest in +psutil (see :gh:`2446`). The two advantages I cared about most were plain +``assert`` statements and +`pytest-xdist `__'s free parallelism. +Tests still inherit from :class:`unittest.TestCase` and there's no +``conftest.py`` or fixture use (rationale in :pr:`2456`). + +What I want to focus on here is one of pytest's most frustrating aspects: slow +startup. + +pytest invocation is slow +------------------------- + +To measure pytest's startup time, let's run a very `simple +test `__ +where execution time itself is negligible: + +.. code-block:: text + + $ time python3 -m pytest psutil/tests/test_misc.py::TestMisc::test_version + 1 passed in 0.05s + real 0m0,427s + +Almost half a second, which is excessive for something I run repeatedly during +development. For comparison, the same test under :mod:`unittest`: + +.. code-block:: text + + $ time python3 -m unittest psutil.tests.test_misc.TestMisc.test_version + Ran 1 test in 0.000s + real 0m0,204s + +Roughly twice as fast. Why? + +Where is time being spent? +-------------------------- + +A significant portion of pytest's overhead comes from import time, and there's +not much one can do about it: + +.. code-block:: text + + $ time python3 -c "import pytest" + real 0m0,151s + + $ time python3 -c "import unittest" + real 0m0,065s + + $ time python3 -c "import psutil" + real 0m0,056s + +Disable plugin auto loading +--------------------------- + +After some research, I discovered that pytest automatically loads all plugins +installed on the system, even if they aren't used. Here's how to list them +(output is cut): + +.. code-block:: text + + $ pytest --trace-config --collect-only + ... + active plugins: + ... + setupplan : ~/.local/lib/python3.12/site-packages/_pytest/setupplan.py + stepwise : ~/.local/lib/python3.12/site-packages/_pytest/stepwise.py + warnings : ~/.local/lib/python3.12/site-packages/_pytest/warnings.py + logging : ~/.local/lib/python3.12/site-packages/_pytest/logging.py + reports : ~/.local/lib/python3.12/site-packages/_pytest/reports.py + python_path : ~/.local/lib/python3.12/site-packages/_pytest/python_path.py + unraisableexception : ~/.local/lib/python3.12/site-packages/_pytest/unraisableexception.py + threadexception : ~/.local/lib/python3.12/site-packages/_pytest/threadexception.py + faulthandler : ~/.local/lib/python3.12/site-packages/_pytest/faulthandler.py + instafail : ~/.local/lib/python3.12/site-packages/pytest_instafail.py + anyio : ~/.local/lib/python3.12/site-packages/anyio/pytest_plugin.py + pytest_cov : ~/.local/lib/python3.12/site-packages/pytest_cov/plugin.py + subtests : ~/.local/lib/python3.12/site-packages/pytest_subtests/plugin.py + xdist : ~/.local/lib/python3.12/site-packages/xdist/plugin.py + xdist.looponfail : ~/.local/lib/python3.12/site-packages/xdist/looponfail.py + ... + +It turns out ``PYTEST_DISABLE_PLUGIN_AUTOLOAD`` environment variable can be +used to disable them. By running +``PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest --trace-config --collect-only`` again +I can see that the following plugins disappeared: + +.. code-block:: text + + anyio + pytest_cov + pytest_instafail + pytest_subtests + xdist + xdist.looponfail + +Now let's run the test again with ``PYTEST_DISABLE_PLUGIN_AUTOLOAD``: + +.. code-block:: text + + $ time PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python3 -m pytest psutil/tests/test_misc.py::TestMisc::test_version + 1 passed in 0.05s + real 0m0,285s + +We went from 0.427s to 0.285s, a ~40% improvement. Not bad. We now need to +selectively enable only the plugins we actually use, via ``-p``. psutil uses +``pytest-instafail`` and ``pytest-subtests`` (we'll deal with ``pytest-xdist`` +later): + +.. code-block:: text + + $ time PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python3 -m pytest -p instafail -p subtests ... + real 0m0,320s + +Time went back up to 0.320s. Quite a slowdown, but still better than the +original 0.427s. Adding ``pytest-xdist``: + +.. code-block:: text + + $ time PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python3 -m pytest -p instafail -p subtests -p xdist ... + real 0m0,369s + +0.369s. Not much, but still a pity to pay the price when NOT running tests in +parallel. + +Handling pytest-xdist +--------------------- + +If we disable ``pytest-xdist`` psutil tests still run, but we get a warning: + +.. code-block:: text + + psutil/tests/test_testutils.py:367 + ~/svn/psutil/psutil/tests/test_testutils.py:367: PytestUnknownMarkWarning: Unknown pytest.mark.xdist_group - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.xdist_group(name="serial") + +This warning appears for methods that are intended to run serially, those +decorated with ``@pytest.mark.xdist_group(name="serial")``. However, since +``pytest-xdist`` is now disabled, the decorator no longer exists. To address +this, I implemented the following solution in ``psutil/tests/__init__.py``: + +.. code-block:: python + + import pytest, functools + + PYTEST_PARALLEL = "PYTEST_XDIST_WORKER" in os.environ # True if running parallel tests + + if not PYTEST_PARALLEL: + def fake_xdist_group(*_args, **_kwargs): + """Mimics `@pytest.mark.xdist_group` decorator. No-op: it just + calls the test method or return the decorated class.""" + def wrapper(obj): + @functools.wraps(obj) + def inner(*args, **kwargs): + return obj(*args, **kwargs) + + return obj if isinstance(obj, type) else inner + + return wrapper + + pytest.mark.xdist_group = fake_xdist_group # monkey patch + +With this in place the warning disappears when running tests serially. To run +tests in parallel, we'll manually enable ``xdist``: + +.. code-block:: text + + $ python3 -m pytest -p xdist -n auto --dist loadgroup + +Optimizing test collection time +------------------------------- + +By default, pytest searches the entire directory for tests, adding unnecessary +overhead. In ``pyproject.toml`` you can tell pytest where test files are +located, and only to consider ``test_*.py`` files: + +.. code-block:: toml + + [tool.pytest.ini_options] + testpaths = ["psutil/tests/"] + python_files = ["test_*.py"] + +Collection time dropped from 0.20s to 0.17s, another ~0.03s shaved off. + +Putting it all together +----------------------- + +With these small optimizations, I managed to reduce ``pytest`` startup time by +~0.12 seconds, bringing it down from 0.42 seconds. While this improvement is +insignificant for full test runs, it makes a noticeable difference (~28% +faster) when repeatedly running individual tests from the command line, which +is something I do frequently during development. Final result is visible in +:pr:`2538`. + +Other links which may be useful +------------------------------- + +- https://github.com/zupo/awesome-pytest-speedup +- https://projects.gentoo.org/python/guide/pytest.html diff --git a/docs/blog/2025/wheels-for-free-threaded-python.rst b/docs/blog/2025/wheels-for-free-threaded-python.rst new file mode 100644 index 0000000000..17800c16ad --- /dev/null +++ b/docs/blog/2025/wheels-for-free-threaded-python.rst @@ -0,0 +1,102 @@ +.. post:: 2025-10-25 + :tags: wheels, community, python-core + :author: Giampaolo Rodola + :exclude: + + Unlocking ``pip install psutil`` for no-GIL Python + +Wheels for free-threaded Python now available +============================================= + +With the release of psutil 7.1.2, wheels for free-threaded Python are now +available. This milestone was achieved largely through a community effort, as +several internal refactorings to the C code were required to make it possible +(see :gh:`2565`). Many of these changes were contributed by +`Lysandros Nikolaou `__. Thanks to him for the +effort and for bearing with me in code reviews! ;-) + +What is free-threaded Python? +----------------------------- + +Free-threaded Python (available since Python 3.13) refers to Python builds that +are compiled with the **GIL (Global Interpreter Lock) disabled**, allowing true +parallel execution of Python bytecodes across multiple threads. This is +particularly beneficial for CPU-bound applications, as it enables better +utilization of multi-core processors. + +The state of free-threaded wheels +--------------------------------- + +According to Hugo van Kemenade's `free-threaded wheels +tracker `__, the +adoption of free-threaded wheels among the top 360 most-downloaded +PyPI packages with C extensions is still limited. Only 128 out of +these 360 packages provide wheels compiled for free-threaded Python, +meaning they can run on Python builds with the GIL disabled. This shows +that, while progress has been made, most popular packages with C +extensions still do not offer ready-made wheels for free-threaded +Python. + +What it means for users +----------------------- + +When a library author provides a wheel, users can install a pre-compiled binary +package without having to build it from source. This is especially important +for packages with C extensions, like psutil, which is largely written in C. +Such packages often have complex build requirements and require installing a C +compiler. On Windows, that means installing Visual Studio or the Build Tools, +which can take several gigabytes and a *significant* setup effort. Providing +wheels spares users from this hassle, makes installation far simpler, and is +effectively essential for the users of that package. You basically +``pip install psutil`` and you're done. + +What it means for library authors +--------------------------------- + +Currently, **universal wheels for free-threaded Python do not exist**. Each +wheel must be built specifically for a Python version. Right now authors must +create separate wheels for Python 3.13 and 3.14. Which means distributing +*a lot* of files already: + +.. code-block:: none + + psutil-7.1.2-cp313-cp313t-macosx_10_13_x86_64.whl + psutil-7.1.2-cp313-cp313t-macosx_11_0_arm64.whl + psutil-7.1.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl + psutil-7.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + psutil-7.1.2-cp313-cp313t-win_amd64.whl + psutil-7.1.2-cp313-cp313t-win_arm64.whl + psutil-7.1.2-cp314-cp314t-macosx_10_15_x86_64.whl + psutil-7.1.2-cp314-cp314t-macosx_11_0_arm64.whl + psutil-7.1.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl + psutil-7.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + psutil-7.1.2-cp314-cp314t-win_amd64.whl + psutil-7.1.2-cp314-cp314t-win_arm64.whl + +This also multiplies CI jobs and slows down the test matrix (see +`build.yml `__). +A true universal wheel would greatly reduce this overhead, allowing a single +wheel to support multiple Python versions and platforms. Hopefully, Python 3.15 +will simplify this process. Two competing proposals, +`PEP 803 `__ and +`PEP 809 `__, aim to standardize +wheel naming and metadata to allow producing a single wheel that covers +multiple Python versions. That would drastically reduce distribution complexity +for library authors, and it's fair to say it's essential for free-threaded +CPython to truly succeed. + +How to install free-threaded psutil +----------------------------------- + +You can now install psutil for free-threaded Python directly via ``pip``: + +.. code-block:: bash + + pip install psutil --only-binary=:all: + +This ensures you get the pre-compiled wheels without triggering a source build. + +Discussion +---------- + +- `Reddit `__ diff --git a/docs/blog/2026/event-driven-process-waiting.rst b/docs/blog/2026/event-driven-process-waiting.rst new file mode 100644 index 0000000000..d1d242c024 --- /dev/null +++ b/docs/blog/2026/event-driven-process-waiting.rst @@ -0,0 +1,261 @@ +.. post:: 2026-01-28 + :tags: python-core, performance, featured + :author: Giampaolo Rodola + :exclude: + + Replacing :meth:`subprocess.Popen.wait`'s busy-loop with an event-driven model, in psutil then upstream CPython + +From Python 3.3 to today: ending 15 years of subprocess polling +=============================================================== + +One of the less fun aspects of process management on POSIX systems is waiting +for a process to terminate. The standard library's :mod:`subprocess` module has +relied on a busy-loop polling approach since the *timeout* parameter was added +to :meth:`subprocess.Popen.wait` in Python 3.3, around 15 years ago (see +`source `__). +And psutil's :meth:`Process.wait` method uses exactly the same technique (see +`source `__). + +The logic is straightforward: check whether the process has exited using +non-blocking ``waitpid(WNOHANG)``, sleep briefly, check again, sleep a bit +longer, and so on. + +.. code-block:: python + + import os, time + + def wait_busy(pid, timeout): + end = time.monotonic() + timeout + interval = 0.0001 + while time.monotonic() < end: + pid_done, _ = os.waitpid(pid, os.WNOHANG) + if pid_done: + return + time.sleep(interval) + interval = min(interval * 2, 0.04) + raise TimeoutExpired + +In this blog post I'll show how I finally addressed this long-standing +inefficiency, first in psutil, and most excitingly, directly in CPython's +standard library :mod:`subprocess` module. + +The problem with busy-polling +----------------------------- + +- CPU wake-ups: even with exponential backoff (starting at 0.1ms, capping at + 40ms), the system constantly wakes up to check process status, wasting CPU + cycles and draining batteries. +- Latency: there's always a gap between when a process actually terminates and + when you detect it. +- Scalability: monitoring many processes simultaneously magnifies all of the + above. + +Event-driven waiting +-------------------- + +All POSIX systems provide at least one mechanism to be notified when a file +descriptor becomes ready. These are :manpage:`select(2)`, :manpage:`poll(2)`, +:manpage:`epoll(7)` (Linux) and +`kqueue() `__ (BSD / macOS) +system calls. Until recently, I believed they could only be used with file +descriptors referencing sockets, pipes, etc., but it turns out they can also be +used to wait for events on process PIDs! + +Linux +----- + +In 2019, Linux 5.3 introduced a new syscall, :func:`os.pidfd_open`, which was +added in Python 3.9. It returns a :term:`file descriptor` referencing a process +PID. The interesting thing is that :manpage:`pidfd_open(2)` can be used in +conjunction with :manpage:`select(2)`, :manpage:`poll(2)` or +:manpage:`epoll(7)` to effectively wait until the process exits. E.g. by using +``poll()``: + +.. code-block:: python + + import os, select + + def wait_pidfd(pid, timeout): + pidfd = os.pidfd_open(pid) + poller = select.poll() + poller.register(pidfd, select.POLLIN) + # block until process exits or timeout occurs + events = poller.poll(timeout * 1000) + if events: + return + raise TimeoutError + +This approach has zero busy-looping. The kernel wakes us up exactly when the +process terminates or when the timeout expires if the PID is still alive. + +I chose ``poll()`` over ``select()`` because ``select()`` has a historical file +descriptor limit (``FD_SETSIZE``), which typically caps it at 1024 +:term:`file descriptors ` per-process (reminded me of +:bpo:`1685000`). + +I chose ``poll()`` over ``epoll()`` because it does not require creating an +additional :term:`file descriptor`. It also needs only a single syscall, which +should make it a bit more efficient when monitoring a single FD rather than +many. + +macOS and BSD +------------- + +BSD-derived systems (including macOS) provide the ``kqueue()`` syscall. It's +conceptually similar to ``select()``, ``poll()`` and ``epoll()``, but more +powerful (e.g. it can also handle regular files). ``kqueue()`` can be passed a +PID directly, and it will return once the PID disappears or the timeout +expires: + +.. code-block:: python + + import select + + def wait_kqueue(pid, timeout): + kq = select.kqueue() + kev = select.kevent( + pid, + filter=select.KQ_FILTER_PROC, + flags=select.KQ_EV_ADD | select.KQ_EV_ONESHOT, + fflags=select.KQ_NOTE_EXIT, + ) + # block until process exits or timeout occurs + events = kq.control([kev], 1, timeout) + if events: + return + raise TimeoutError + +Windows +------- + +Windows does not busy-loop, both in psutil and :mod:`subprocess` module, thanks +to ``WaitForSingleObject``. This means Windows has effectively had event-driven +process waiting from the start. So nothing to do on that front. + +Graceful fallbacks +------------------ + +Both ``pidfd_open()`` and ``kqueue()`` can fail for different reasons. For +example, with ``EMFILE`` if the process runs out of +:term:`file descriptors ` (usually 1024), or with ``EACCES`` / +``EPERM`` if the syscall was explicitly blocked at the system level by the +sysadmin (e.g. via SECCOMP). In all cases, psutil silently falls back to the +traditional busy-loop polling approach rather than raising an exception. + +This fast-path-with-fallback approach is similar in spirit to :bpo:`33671`, +where I sped up :func:`shutil.copyfile` by using zero-copy system calls back in +2018. In there, more efficient :func:`os.sendfile` is attempted first, and if +it fails (e.g. on network filesystems) we fall back to the traditional +:manpage:`read(2)` / :manpage:`write(2)` approach to copy regular files. + +Measurement +----------- + +As a simple experiment, here's a simple program which waits on itself for 10 +seconds without terminating: + +.. code-block:: python + + # test.py + import psutil, os + try: + psutil.Process(os.getpid()).wait(timeout=10) + except psutil.TimeoutExpired: + pass + +We can measure the CPU context switching using ``/usr/bin/time -v``. Before the +patch (the busy-loop): + +:: + + $ /usr/bin/time -v python3 test.py 2>&1 | grep context + Voluntary context switches: 258 + Involuntary context switches: 4 + +After the patch (the event-driven approach): + +:: + + $ /usr/bin/time -v python3 test.py 2>&1 | grep context + Voluntary context switches: 2 + Involuntary context switches: 1 + +This shows that instead of spinning in userspace, the process blocks in +``poll()`` / ``kqueue()``, and is woken up only when the kernel notifies it, +resulting in just a few CPU :term:`context switches `. + +Sleeping state +-------------- + +It's also interesting to note that waiting via ``poll()`` (or ``kqueue()``) +puts the process into the exact same sleeping state as a plain +:func:`time.sleep` call. From the kernel's perspective, both are interruptible +sleeps: the process is de-scheduled, consumes zero CPU, and sits quietly in +kernel space. + +The ``"S+"`` state shown below by ``ps`` means that the process “sleeps in +foregroundâ€. + +- :func:`time.sleep`: + +:: + + $ (python3 -c 'import time; time.sleep(10)' & pid=$!; sleep 0.3; ps -o pid,stat,comm -p $pid) && fg &>/dev/null + PID STAT COMMAND + 491573 S+ python3 + +- :func:`select.poll`: + +:: + + $ (python3 -c 'import os,select; fd = os.pidfd_open(os.getpid(),0); p = select.poll(); p.register(fd,select.POLLIN); p.poll(10_000)' & pid=$!; sleep 0.3; ps -o pid,stat,comm -p $pid) && fg &>/dev/null + PID STAT COMMAND + 491748 S+ python3 + +CPython contribution +-------------------- + +After landing the psutil implementation (:pr:`2706`), I took the extra step and +submitted a matching pull request for CPython :mod:`subprocess` module: +:cpy-pr:`144047`. + +I'm especially proud of this one: this is the **third time** in psutil's 17+ +year history that a feature developed in psutil made its way upstream into the +Python standard library. + +- The first was back in 2010, when :meth:`Process.nice` inspired + :func:`os.getpriority` and :func:`os.setpriority`, see :bpo:`10784`. Landed + in Python 3.3. + +- The second was back in 2011, when :func:`psutil.disk_usage` inspired + :func:`shutil.disk_usage`, see `python-ideas ML + proposal `__. + Landed in Python 3.3. + +*Funny thing:* 15 years ago, Python 3.3 added the *timeout* parameter to +:meth:`subprocess.Popen.wait` (see +`commit `__). That's +probably where I took inspiration when I first added the *timeout* parameter to +psutil's :meth:`Process.wait` around the same time (see :commit:`886710daf`). +Now, 15 years later, I'm contributing back a similar improvement for that very +same *timeout* parameter. **The circle is complete**. + +Links +----- + +Topics related to this: + +- :gh:`2712`: proposal to extend this to multiple PIDs + (:func:`psutil.wait_procs`). +- :gh:`2703`: proposal for asynchronous :meth:`Process.wait` integration with + :mod:`asyncio`. +- :cpy:`144211`: proposal to extend the :mod:`selectors` module to enable + :mod:`asyncio` optimization on BSD / macOS via ``kqueue()``. + +Discussion +---------- + +- `Reddit `__ +- `Hacker News `__ +- `Medium `__ +- `Linkedin `__ diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 0000000000..6a2f0a7dbb --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1,3625 @@ +.. + Sections a release can use, in this order. In brackets is the GitHub + label each maps to. + + - New APIs [new-api]: a new function, argument or field, or an existing + one now working where it didn't. + - New platforms [new-platform]: an OS, architecture or interpreter. + - API changes [api-change]: an API changed, was deprecated or removed. + - Performance [performance]: something got faster. + - Build and packaging [packaging]: wheels, sdist, what gets installed. + - Documentation [doc]: docs/ only. + - Internals [internals, ci, tests, scripts]: psutil's own machinery, + debug output included. Nothing user-facing. + - Dropped support [dropped-support]: a platform, OS or Python version. + - Bug fixes [bug]: crashes, wrong results, leaks, build failures. + + Format: ``- :gh:`N`, [Platform], :label:`name`: text.`` Labels are + ``breaking``, ``critical``, ``build-fail`` and ``memleak``, from the + compatibility, critical, build-fail and memleak labels on the tracker. + Platform tags come from the platform labels. + + 8.0.0 splits Bug fixes per platform, and has its own Other API changes + and Type hints and enums sections. + +Changelog +========= + +8.0.0 (IN DEVELOPMENT) +^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + psutil 8.0 introduces breaking API changes. See the + :ref:`migration guide ` if upgrading from 7.x. + +**New APIs** + +- :gh:`2798`, :label:`breaking`: new :attr:`Process.attrs` class attribute, a + :class:`frozenset` of the attribute names accepted by :meth:`Process.as_dict` + and :func:`process_iter`. Passing ``attrs=[]`` to :func:`process_iter` to + mean "retrieve all attributes" is deprecated. See + :ref:`migration guide `. +- :gh:`2776`, [Windows], :label:`breaking`: :func:`virtual_memory` now includes + :field:`cached` and :field:`wired` fields. +- :gh:`1541`: New :meth:`Process.page_faults` method, returning a + ``(minor, major)`` named tuple. +- :gh:`2780`, [Windows]: :func:`disk_usage` now can accept a file path (not + only a directory path). +- :gh:`2816`, [OpenBSD]: :func:`swap_memory` :field:`sin` and :field:`sout` are + no longer set to ``0``. +- :gh:`2977`: new :func:`bytes2human` utility function, converting a number of + bytes to a human-readable string (e.g. ``9.8K``). + +Reorganization of process memory APIs (:gh:`2731`, :gh:`2736`, :gh:`2723`, +:gh:`2733`, :gh:`2988`). + +- New :meth:`Process.memory_extras` method, returning extra platform-specific + memory metrics on Linux, macOS and Windows. See + :ref:`migration guide `. + +- New :meth:`Process.memory_footprint` method, which returns :field:`uss`, + :field:`pss` and :field:`swap` metrics (what :meth:`Process.memory_full_info` + used to return). + +- :label:`breaking`: :meth:`Process.memory_full_info` is deprecated. Use the + new :meth:`Process.memory_footprint` instead. See + :ref:`migration guide `. + +- :label:`breaking`: :meth:`Process.memory_info` named tuple changed on all + platforms: fields were added, removed and renamed. Most old names still work + but raise :exc:`DeprecationWarning`; on macOS :field:`pfaults` and + :field:`pageins` were removed with **no backward-compatible aliases**, use + :meth:`Process.page_faults` instead. See + :ref:`migration guide `. + +**Type hints and enums** + +- :gh:`2753`, :label:`breaking`: Introduce enum classes + (:class:`ProcessStatus`, :class:`ConnectionStatus`, + :class:`ProcessIOPriority`, :class:`ProcessPriority`, :class:`ProcessRlimit`) + grouping related constants. The top-level constants (e.g. + :data:`STATUS_RUNNING`) remain the primary API, and are now aliases for the + corresponding enum members. See :ref:`migration guide `. +- :gh:`1946`: Add inline type hints to all public APIs in + ``psutil/__init__.py``. Editors and checkers that read inline annotations + (pyright, Pylance) pick them up automatically. mypy ignores them until we + ship a ``py.typed`` marker, so it still relies on the third-party + ``types-psutil`` stubs. No runtime behavior is changed. +- :gh:`2751`: Convert all named tuples from :func:`collections.namedtuple` to + :class:`typing.NamedTuple` classes with **type annotations**. This makes the + classes self-documenting, effectively turning this module into a readable API + reference. + +**Other API changes** + +- :gh:`2747`, :label:`breaking`: the field order of the named tuple returned by + :func:`cpu_times` has been normalized on all platforms, and the first 3 + fields are now always :field:`user`, :field:`system`, :field:`idle`. See + :ref:`migration guide `. +- :gh:`2772`, [Windows], :label:`breaking`: :func:`cpu_times` + :field:`interrupt` field renamed to :field:`irq` to match the field name used + on Linux and BSD. :field:`interrupt` still works but raises + :exc:`DeprecationWarning`. +- :gh:`2784`, :label:`breaking`: :func:`process_iter`: when *attrs* is + specified, the pre-fetched values are now cached on the :class:`Process` + instance, so subsequent method calls return them without new system calls. + The ``p.info`` dict is deprecated. See + :ref:`migration guide `. +- :gh:`2889`, [Windows], :label:`breaking`: 32-bit psutil can no longer inspect + 64-bit processes. This relied on the undocumented ``NtWow64*`` APIs and + stopped being tested when 32-bit wheels were dropped in 7.1.2 (:gh:`2657`). + :meth:`Process.cwd` now raises :exc:`AccessDenied` in that case; + :meth:`Process.cmdline` and :meth:`Process.environ` are unaffected. The + opposite direction (64-bit psutil inspecting 32-bit processes) still works. +- :gh:`2754`: standardize :func:`sensors_battery`'s :field:`percent` so that it + returns a ``float`` instead of ``int`` on all systems, not only Linux. +- :gh:`2799`: :meth:`Process.as_dict` now returns a dict with keys sorted + alphabetically when *attrs* is not specified. +- :gh:`2805`, [BSD]: remove ``procfs`` dependency on NetBSD for + :func:`cpu_stats` and :func:`virtual_memory`; values are now retrieved via + the ``sysctl(9)`` and ``uvm(9)`` kernel APIs instead. (patch by + :user:`Santhosh Raju `) +- :gh:`2947`: psutil now emits a ``RuntimeWarning`` when it returns incomplete + or approximated results due to an unexpected condition (e.g. a sanity check + on kernel data which failed). Before, these events were only visible by + enabling debug mode via the :envvar:`PSUTIL_DEBUG` environment variable, so + in practice they went unnoticed. + +**Performance** + +- :gh:`2695`, [Windows]: :func:`net_io_counters` is **~5x faster**. + ``GetAdaptersAddresses()`` is now invoked once instead of twice, and it skips + collecting unicast / anycast / multicast / DNS details, which were retrieved + but never used. :func:`net_if_stats` and :func:`net_if_addrs` also got + faster. (patch by :user:`Arman Luthra `) +- :gh:`2919`, :gh:`2920`, [Windows]: :meth:`Process.threads` is around + **25x faster**, :meth:`Process.ppid` and :meth:`Process.children` around + 3.5x. They no longer snapshot every thread / process on the system with + ``CreateToolhelp32Snapshot``, and read + ``NtQuerySystemInformation(SystemProcessInformation)`` in one shot instead. + As a side effect :meth:`Process.threads` no longer silently misses the + threads which could not be opened due to :exc:`AccessDenied`. +- :gh:`2922`, [Windows]: :meth:`Process.ppid` is around **58x faster** (**99x** + on ARM64). Instead of fetching the whole process table to read one field, the + parent PID is now read from + ``NtQueryInformationProcess(ProcessBasicInformation)``. +- :gh:`2923`, [Windows]: :meth:`Process.status` is now part of the + :meth:`Process.oneshot` group, so within that context (also used by + :meth:`Process.as_dict`) it no longer costs an extra system-wide query. + Reading 4 methods of that group in one :meth:`Process.oneshot` block is now + around **3.9x faster** than reading them without it, up from 2x. +- :gh:`2932`, [Windows]: :meth:`Process.open_files` is **140x to 400x faster** + (from 235 ms to 0.39 ms per call). It no longer enumerates every handle in + the system with + ``NtQuerySystemInformation(SystemExtendedHandleInformation)``, but + per-process, via ``NtQueryInformationProcess(ProcessHandleInformation)``, and + the ones which are not files are skipped by object type index, before being + duplicated. Also, the internal thread used to query handle names is now + created once per call instead of once per handle. +- :gh:`2939`: syscalls which can potentially block (disk devices, mount points, + NIC drivers, etc) now release the GIL. Before, a slow psutil call would + freeze all the other threads of the application for its whole duration. + +**Build and packaging** + +- :gh:`2976`, [Linux]: publish ppc64le and s390x wheels. +- :gh:`2788`, :label:`breaking`: git tags renamed from ``release-X.Y.Z`` to + ``vX.Y.Z``. Old tags are kept for backward compatibility. See + :ref:`migration guide `. +- :gh:`2914`, [macOS], :label:`breaking`: Intel wheels now require macOS 10.15 + (Catalina) or higher, up from 10.9. Older versions account for 0.01% of macOS + downloads, and they must now build from source. ``arm64`` wheels are + unaffected: they already required macOS 11. +- :gh:`2915`, :label:`breaking`: stop publishing wheels for free-threaded + CPython 3.13 (``cp313t``). Free-threading was experimental in 3.13 and is + officially not recommended. Publish only ``cp314t`` wheels. +- :gh:`2576`: the C extension modules now use PEP 489 multi-phase + initialization instead of single-phase, which is the preferred mechanism for + extension modules. Runtime behavior is unchanged. +- :gh:`2765`: add a PR bot that uses Claude to summarize PR changes and update + ``changelog.rst`` and ``credits.rst`` when commenting with /changelog. +- :gh:`2766`: remove remaining Python 2.7 compatibility shims from + ``setup.py``, simplifying the build infrastructure. +- :gh:`2844`: removed docs/ from tarball. Tarball before: 586K. Tarball now: + 396K. +- :gh:`2883`: the platform-specific C extension modules (``_psutil_linux``, + ``_psutil_windows``, etc.) are now built as a single private module named + ``_psutil`` on all platforms. +- :gh:`2909`: ``setup.py`` no longer uses ``distutils``, which was removed from + the stdlib in Python 3.12, and only relies on ``setuptools``. +- :gh:`2925`: the C sources are now compiled in parallel, making builds + **2x to 3.6x faster**. This mostly benefits the platforms getting no wheels + from PyPI (\*BSD, Solaris, AIX), where ``pip install psutil`` always + compiles. Use :envvar:`PSUTIL_BUILD_JOBS` to cap the number of jobs. +- :gh:`2927`: python dependencies (``make install-pydeps-*``) are now installed + with ``uv`` when available, saving around 10 secs for each CI run. + +**Documentation** + +- :gh:`2757`, :gh:`2760`: split docs from a single HTML file into multiple new + sections: :doc:`/about `, :doc:`/adoption ` (:gh:`2763`), + :doc:`/alternatives ` (:gh:`2775`), + :doc:`/api-overview `, :doc:`/credits ` (:gh:`2764`), + :doc:`/faq ` (:gh:`2769`), :doc:`/funding ` (:gh:`2797`), + :ref:`/genindex ` (:gh:`2808`), :doc:`/glossary ` + (:gh:`2774`), :doc:`/install `, :doc:`/migration ` + (:gh:`2771`), :doc:`/performance ` (:gh:`2787`), + :doc:`/platform `, :doc:`/recipes ` (:gh:`2761`), + :doc:`/shell-equivalents ` (:gh:`2768`), + :doc:`/stdlib-equivalents ` (:gh:`2781`). The old + ``INSTALL.rst`` and ``CREDITS`` files, which lived in the root dir, moved + there as well. + +- Blog: new blog at :doc:`/blog ` (:gh:`2825`), built via the + `ablog `__ Sphinx extension, with 20 posts + imported from https://gmpy.dev, covering psutil topics from 2014 to 2026. + Posts are searchable, have an Atom feed, use OpenGraph for a nice preview + when shared on social media, and have a comments section backed by + `giscus `__ and GitHub Discussions (:gh:`2879`). + +- Theming: renewed, modern, custom theme, with a top bar (:gh:`2819`), a + toggable dark theme (:gh:`2803`, ``Shift+D``), Monokai code snippets, a "last + updated" stamp in the footer and an icon marking external URLs. + +- Usability: right TOC sidebar (:gh:`2828`), a COPY button on code snippets + (:gh:`2761`), a "copy page" button copying the page's RsT source + (:gh:`2981`), a "back to top" button, the ``psutil.`` prefix shown for all + APIs, and search results styled as cards. Identifiers in code blocks (e.g. + ``psutil.Process()``, ``p.cpu_percent()``) are now clickable and link to + their API reference entry, via + `sphinx-codeautolink `__ + (:gh:`2826`). Doc clarity was improved and long sentences shortened + (:gh:`2745`, :gh:`2801`). New keyboard shortcuts (:gh:`2820`): ``?`` shows + the helper, ``Shift+D`` toggles dark/light mode, ``Ctrl+K`` focuses the + search box, ``Up``/``Down`` navigate the search results and ``Enter`` opens + the selected one. + +- Testing: ``rstcheck`` was replaced by ``sphinx-lint``, plus a custom script + detecting dead reference links in ``.rst`` files (:gh:`2767`). Python code + snippets are syntax-checked at build time (:gh:`2761`). New + ``make test-docs`` with sanity checks for the built HTML docs. + +- Hosting: doc is no longer hosted on Read the Docs. It's now self-hosted on + GitHub Pages under a new domain, https://psutil.io, and URLs no longer carry + the ``/en`` language and version prefixes, e.g. https://psutil.io/faq/ + (:gh:`2790`). It's rebuilt and deployed automatically on every push to + ``master`` (:gh:`2739`). Past releases are also served, e.g. + https://psutil.io/7.2/, with a version selector in the top bar (:gh:`2980`). + +- Misc: doc is built as part of CI (fails on error), all ``.rst`` files are + wrapped to 79 characters via https://github.com/giampaolo/rstwrap + (:gh:`2823`), and ``/sitemap.xml`` was added to help search engine discovery. + New custom 404 page (:gh:`2829`): hovering over the © copyright in the footer + reveals an easter egg which takes you there. + +**Dropped support** + +- :gh:`2872`, :label:`breaking`: Dropped support for Python 3.6 and 3.7. + Minimum version is now 3.8. +- :gh:`2893`, [Windows], :label:`breaking`: Dropped support for Windows Vista, + 7, 8, 8.1 and their server counterparts. Minimum version is now Windows 10 / + Server 2016. See :ref:`migration guide `. +- :gh:`2936`, [Windows], :label:`breaking`: dropped support for PyPy older than + 7.3.14 (December 2023). +- :gh:`2987`, [macOS], :label:`breaking`: dropped support for macOS 10.7 and + 10.8. Minimum version is now 10.9. + +**Bug fixes: cross-platform** + +- :gh:`2793`: :func:`process_iter` was silently dropping zombie processes + because :exc:`ZombieProcess` (a subclass of :exc:`NoSuchProcess`) was caught + by the wrong ``except`` clause. Zombie processes are now yielded correctly. +- :gh:`2895`: :class:`Process` methods could wrongly raise :exc:`NoSuchProcess` + ("PID has been reused") when the process creation time could not be + determined, e.g. for zombies on NetBSD / OpenBSD or on :exc:`AccessDenied` on + Windows. An unknown creation time is no longer treated as proof of PID reuse. +- :gh:`2899`: two :class:`Process` instances for the same process could compare + unequal if the creation time of either one could not be determined. They now + compare equal, and ``hash(Process)`` is based on the PID alone. + +**Bug fixes: Linux** + +- :gh:`2860`, :gh:`2966`, [Linux], :label:`critical`: + :meth:`Process.cpu_affinity` could crash the interpreter with a segfault when + ``PyLong_FromLong()`` returned NULL under memory pressure (the NULL is now + checked and a proper :exc:`MemoryError` is raised instead), and on Python <= + 3.11 it over-decref'ed the CPU numbers it returned, corrupting CPython's + small integer cache. +- :gh:`2857`, [Linux], [SunOS], :label:`memleak`: fix refcount leak in + ``disk_partitions()`` (Linux) and ``proc_environ()`` (SunOS) when + ``PyArg_ParseTuple`` fails: parse arguments before allocating the result + container, matching the pattern used in the other 26 call sites. Also fix a + copy-paste typo in SunOS ``proc_environ()`` where the post-decode NULL check + examined the wrong variable (``py_envname`` instead of ``py_envval``), which + could let a NULL value reach ``PyDict_SetItem``. +- :gh:`2628`, [Linux]: :func:`cpu_freq` no longer takes offline CPU cores into + account. They were reported with all-zero frequencies, which dragged down the + average ``current``, ``min`` and ``max`` values. +- :gh:`2715`, [Linux]: ``wait_pid_pidfd_open()`` (from :meth:`Process.wait`) + crashes with ``EINVAL`` due to kernel race condition. +- :gh:`2732`, [Linux]: :func:`net_if_stats`: handle ``EBUSY`` from + ``ioctl(SIOCETHTOOL)``. +- :gh:`2770`, [Linux]: fix :func:`cpu_count` (``logical=False``) raising + :exc:`ValueError` on s390x architecture, where :proc:`/proc/cpuinfo` uses + spaces before the colon separator instead of a tab. +- :gh:`2809`, [Linux]: :func:`swap_memory` and :func:`virtual_memory` raise + ``ValueError`` if :proc:`/proc/meminfo` contains a field with no space after + the colon, e.g. ``ShadowCallStack:10373888 kB``, which occurs on arm64 when + shadow call stacks exceed 10 GB. +- :gh:`2830`, [Linux], [macOS], [BSD]: :meth:`Process.terminal` returned + ``None`` for terminals opened after the first call, e.g. a new ``/dev/pts/N`` + in a long running daemon. The list of terminal devices was cached forever, + and is now refreshed when it doesn't know a device. +- :gh:`2871`, [Linux]: :meth:`Process.rlimit` returned ``RLIM_INFINITY`` as the + unsigned ``2**64-1`` instead of ``-1`` on Python 3.15+, which changed + ``resource.prlimit()`` accordingly. psutil now maps it back to + :data:`psutil.RLIM_INFINITY` so the value stays consistent across Python + versions. +- :gh:`2967`, [Linux]: :func:`cpu_freq` returned ``None`` on ppc machines + without cpufreq sysfs. On s390x it matched both ``cpu MHz dynamic`` and + ``cpu MHz static``, reporting twice as many CPUs as the machine has. +- :gh:`2512`, [Linux]: :func:`cpu_freq` with ``percpu=True`` returned one entry + per cpufreq policy instead of one per CPU, so on hardware where a policy is + shared by several CPUs (POWER9, Apple M1, RISC-V) it reported fewer entries + than :func:`cpu_count`. Each policy is now asked which CPUs it affects. + (patch by Julien Stephan) +- :gh:`2611`: re-enable support for Android, which got broken on Python 3.13. + Also include a working :func:`disk_partitions` which no longer raises + ``PermissionError``. + +**Bug fixes: Windows** + +- :gh:`1967`, [Windows], :label:`critical`: :meth:`Process.open_files` could + deadlock the calling process. On timeout, the internal thread querying a + handle name was killed with ``TerminateThread()``, which cannot terminate a + thread blocked in the kernel (e.g. on a pipe with a pending read) and left + locks and memory in an inconsistent state. The thread is now abandoned and + cleans up after itself. +- :gh:`2859`, [Windows], :label:`critical`: :func:`net_connections` / + :meth:`Process.net_connections` could crash with an invalid + ``Py_DECREF(NULL)`` when argument parsing failed before the result list was + allocated. The error path now uses ``Py_XDECREF`` (including the temporary + address-family / socket-type objects). +- :gh:`2847`, [Windows], :label:`critical`: :func:`cpu_stats` read the context + switches and syscalls counts from a buffer it had just freed. +- :gh:`2934`, [Windows], :label:`critical`: :meth:`Process.memory_maps` could + crash the calling process with a stack buffer overflow if the inspected + process had a mapped file whose path is longer than 260 characters. The + buffer size was passed to ``GetMappedFileNameW()`` in bytes instead of + characters. Also, such paths are now returned in full instead of truncated. +- :gh:`2937`, [Windows], :label:`critical`: :func:`disk_io_counters` could let + a disk driver write past the end of the ``DISK_PERFORMANCE`` buffer. When the + driver asked for more space we retried passing a bigger size, but the buffer + was a fixed size struct on the stack. It is now allocated (and grown) on the + heap. +- :gh:`2943`, [Windows], :label:`critical`: :func:`win_service_iter` could + crash the interpreter instead of raising an exception if the service + enumeration failed. +- :gh:`2946`, [Windows], :label:`critical`: if the number of process heaps + changed while :func:`heap_info` was running, it could read uninitialized + memory and return bogus :field:`mmap_used` and :field:`heap_count` values. +- :gh:`2972`, [Windows], :label:`critical`: on systems with more than 64 CPUs + :func:`cpu_times` with ``percpu=True`` and :func:`cpu_stats` read + uninitialized memory: the kernel only returns entries for the calling + thread's processor group, but the entries for the remaining CPUs were used as + well. +- :gh:`2932`, [Windows], :label:`memleak`: :meth:`Process.open_files` leaked a + thread and a handle for every name query which timed out (e.g. a pipe with a + pending read), and could hang for around 1 minute if the process had a file + open on an unreachable network share, where the :func:`os.stat` used to + filter out directories went over the wire with no timeout. +- :gh:`2935`, [Windows], :label:`memleak`: :meth:`Process.kill` and + :meth:`Process.terminate` leaked a process handle when ``TerminateProcess()`` + failed with an error other than ``ERROR_ACCESS_DENIED``. +- :gh:`1007`, [Windows]: :func:`boot_time` no longer fluctuates by ~1 second + across calls or across processes. It is now read atomically from the kernel + via ``NtQuerySystemInformation(SystemTimeOfDayInformation)``, replacing the + old ``time.time() - uptime()`` computation that sampled two counters from + Python and produced sub-second differences. +- :gh:`1959`, [Windows]: :func:`disk_partitions` could raise + ``UnicodeDecodeError`` on systems whose ANSI code page is not UTF-8 (e.g. + cp1251). It now uses the wide-char Windows APIs throughout. +- :gh:`2383`, [Windows]: :meth:`WindowsService.description` may fail with + ``ERROR_FILE_NOT_FOUND`` when the description points at a missing resource + (e.g. ``WaaSMedicSvc``), which also broke :meth:`WindowsService.as_dict`. Now + it returns an empty string instead. +- :gh:`2655`, [Windows]: :func:`net_if_stats` returned ``4294967295`` (32-bit + overflow) as the speed for network interfaces faster than ~4.29 Gbps (e.g. 5 + Gbps NICs). Fixed by switching from the legacy ``GetIfTable()`` / + ``MIB_IFROW`` API to ``GetIfEntry2()`` / ``MIB_IF_ROW2``, which uses a 64-bit + ``TransmitLinkSpeed`` field. +- :gh:`2711`, :gh:`2940`, [Windows]: :func:`net_if_addrs` was returning + ``None`` for the ``broadcast`` field of network interfaces instead of the + correct broadcast address, and could report an IPv4 :field:`netmask` for an + IPv6 address of the same NIC (the netmask was reset once per interface + instead of once per address). +- :gh:`2875`, [Windows]: :func:`sensors_battery` never returned + :data:`POWER_TIME_UNKNOWN` when the remaining battery time was unknown; it + returned ``4294967295`` instead of ``-1`` due to ``BatteryLifeTime`` being + passed as an unsigned integer. +- :gh:`2938`, [Windows]: :func:`disk_partitions` returned a different + :field:`opts` string for volume mount points than for the drive they live on: + the drive type (``fixed``, ``cdrom``, ...) was missing. +- :gh:`2941`, [Windows]: :func:`net_io_counters` raised :exc:`RuntimeError`, + losing the counters of all the other NICs, if a NIC was disabled or unplugged + mid-call. Now it's skipped. + +**Bug fixes: macOS** + +- :gh:`2885`, [macOS], :label:`critical`: :meth:`Process.memory_full_info`, + :meth:`Process.memory_footprint` and :meth:`Process.threads` no longer use + ``task_for_pid()`` syscall, which can hang forever on headless VMs (e.g. CI + runners). They now use ``proc_pidinfo()``, which is more permissive and so + raises :exc:`AccessDenied` less often. +- :gh:`2382`, [macOS]: :func:`cpu_freq` is now always defined on ARM64 and + returns ``None`` when CPU frequency can't be determined. Previously it was + left undefined (or raised :exc:`RuntimeError`) when the ``pmgr`` IORegistry + entry or its frequency data was unavailable, e.g. on virtualized ARM64 like + CI runners. +- :gh:`2411`, [macOS]: :meth:`Process.cpu_times` and + :meth:`Process.cpu_percent` calculation on macOS x86_64 (arm64 is fine) was + highly inaccurate (41.67x lower). +- :gh:`2642`, [macOS]: fix :func:`cpu_freq` on Apple Silicon. On M4+ it + returned values ~1000x too small because the ``voltage-statesN-sram`` + IORegistry tables switched from Hz to kHz; on M5-family chips it failed + because the hardcoded table indexes were renumbered. The implementation now + enumerates CPU ``voltage-states*-sram`` tables dynamically, detects the unit + per-value by magnitude, and filters CPU clusters from GPU/NPU tables via a + per-table fmax threshold. Works uniformly from M1 through M5 Max. (patch by + Bert Pluymers) +- :gh:`2726`, [macOS]: :meth:`Process.num_ctx_switches` return an unusual high + number due to a C type precision issue. +- :gh:`2841`, [macOS]: :func:`cpu_freq` could raise :exc:`SystemError` when CPU + frequency data is missing or invalid in the IORegistry (e.g. on Apple M5 + chips). It now returns ``None`` instead (see :gh:`2382`). +- :gh:`2853`, [macOS]: :func:`virtual_memory` and :func:`swap_memory` could + raise :exc:`RuntimeError` (``(ipc/mig) array not large enough``) on newer + macOS versions. +- :gh:`2854`, [macOS]: :meth:`Process.cmdline` and :meth:`Process.environ` + could raise :exc:`SystemError` after ``sysctl(KERN_PROCARGS2)`` failed with + ``errno == 0``. They now raise :exc:`AccessDenied` instead. + +**Bug fixes: BSD** + +- :gh:`2744`, [NetBSD], :label:`critical`: fix possible double ``free()`` in + :func:`swap_memory`. +- :gh:`2848`, [BSD], :label:`critical`: fix a stack buffer overflow in + :func:`net_io_counters` when the kernel reports an unusually long interface + name. +- :gh:`1534`, [NetBSD]: :meth:`Process.exe` is now fetched natively via + ``sysctl(KERN_PROC_PATHNAME)`` instead of reading the ``/proc/pid/exe`` + symlink (a virtualization layer on NetBSD). (patch by Kamil Rytarowski) +- :gh:`1801`, [FreeBSD]: :func:`cpu_freq` could raise :exc:`UnicodeDecodeError` + when the ``dev.cpu.N.freq_levels`` sysctl returned bytes which are not valid + UTF-8. +- :gh:`2746`, [FreeBSD]: :meth:`Process.memory_maps`, :field:`rss` and + :field:`private` fields are erroneously reported in memory pages instead of + bytes. Other platforms (Linux, macOS, Windows) return bytes. +- :gh:`2782`, [FreeBSD]: :func:`cpu_count` ``logical=False`` return None on + systems without hyper threading. +- :gh:`2791`, [FreeBSD]: relax ``psutil_sysctl()`` / ``psutil_sysctlbyname()`` + to allow the kernel to return fewer bytes than the buffer (normal for + variable-length ``sysctl`` data). +- :gh:`2795`, [FreeBSD]: fix :func:`cpu_freq` failing with + ``RuntimeError: sysctlbyname('dev.cpu.0.freq_levels') size mismatch`` on some + systems. +- :gh:`2811`, :gh:`2813`, [OpenBSD]: :func:`virtual_memory` :field:`shared` + returned pages instead of bytes, plus it was overvalued (summed shared + ``virtual`` + ``real``, now we only return ``real``). :field:`buffers` was + always 0, and now returns the same value as :field:`cached`, since OpenBSD + does not distinguish between the 2. +- :gh:`2814`, :gh:`2815`, [NetBSD]: :func:`virtual_memory` :field:`cached` is + overvalued, since it includes anonymous pages, and :field:`shared` was + overvalued (summed shared ``virtual`` + ``real``, now we only return + ``real``). +- :gh:`2822`, [BSD]: :meth:`Process.cmdline` on NetBSD could raise + ``OSError: [Errno 14] Bad address`` if the process about to exit. It now + raises :exc:`NoSuchProcess` instead. +- :gh:`2888`, [FreeBSD], [OpenBSD]: :class:`Process` methods could wrongly + raise :exc:`NoSuchProcess` ("PID has been reused") for a process still alive, + after a system clock update (e.g. NTP). Fixed by disabling the PID reuse + check (also on SunOS and AIX). +- :gh:`2902`, [NetBSD], [OpenBSD]: :meth:`Process.cmdline` could fail with + ``OSError(EINVAL)`` for a process which died mid-call (OpenBSD), or raise a + broken :exc:`NoSuchProcess` whose ``str()`` in turn raised :exc:`TypeError` + (NetBSD). +- :gh:`2903`, [BSD]: :meth:`Process.nice` could raise :exc:`NoSuchProcess` for + processes in ``SIDL`` state (not yet fully initialized). It now retrieves the + nice value via ``sysctl()`` instead of ``getpriority()``. +- :gh:`2905`, [FreeBSD], [OpenBSD], [NetBSD]: the ``saved`` field of + :meth:`Process.gids` mistakenly reported the process saved *user* ID instead + of the saved group ID. Bug existed since 2011. +- :gh:`2907`, [NetBSD]: a process which is exiting, but is not a zombie yet, + was not recognized as such. :class:`Process` methods raised + :exc:`NoSuchProcess` instead of :exc:`ZombieProcess`, and + :meth:`Process.status` returned ``"sleeping"`` or ``"?"``. Also, + :data:`STATUS_SUSPENDED` was never returned by :meth:`Process.status`, + despite being documented as NetBSD only, and :meth:`Process.environ` raised + ``OSError`` with ``EINVAL`` / ``EFAULT`` / ``EBUSY`` for a process which is + exiting or is a zombie (it now returns an empty dict, or raises + :exc:`NoSuchProcess` if the process is gone). +- :gh:`2929`, [NetBSD], [OpenBSD]: :meth:`Process.num_fds` returned a wrong, + system-wide number which didn't change when the process opened a file, and + :meth:`Process.open_files` always returned an empty list. +- :gh:`2951`, [OpenBSD]: :func:`cpu_times` returned times averaged across CPUs + instead of summed, like on all the other platforms. Now it sums the per-CPU + counters. +- :gh:`2952`, [OpenBSD]: :meth:`Process.environ` raised ``OSError(EINVAL)`` for + a process which started exiting, e.g. mid-way to becoming a zombie. Also, for + consistency, :meth:`Process.environ` for a zombie now raises + :exc:`ZombieProcess` on all BSDs (NetBSD used to return an empty dict, see + :gh:`2911`). + +**Bug fixes: UNIX** + +- :gh:`2877`, [UNIX], :label:`critical`: fix a one-byte stack buffer overflow + in :func:`users`. When ``ut_host`` fills the whole field it has no null + terminator, and the terminator was written one byte past the end of the local + buffer. +- :gh:`2789`, [AIX], :label:`build-fail`: fix compilation error caused by a + typo (accidental space) in ``psutil_net_io_counters()``, introduced during a + previous code reformatting. +- :gh:`2858`, [SunOS], :label:`memleak`: :func:`disk_io_counters` leaked the + result dictionary when ``kstat_read()`` failed mid-iteration; the error path + now goes through the existing cleanup block. +- :gh:`2687`, [SunOS]: :func:`users` failed with ``ValueError`` on illumos. +- :gh:`2778`, [UNIX]: :func:`net_if_addrs` skips interfaces with no addresses, + which are typically virtual IPv4/IPv6 tunnel interfaces. Now they are + included in the returned dict with :field:`family` == + :data:`socket.AF_UNSPEC` and an empty list of addresses. Main reason: it + creates an inconsistency with :func:`net_io_counters` and + :func:`net_if_stats` which do return these interface names. +- :gh:`2964`, [POSIX]: :func:`net_if_addrs` returned the interface's own + address as :field:`broadcast` for ``/32`` IPv4 addresses. A single-host + network has no broadcast address, so ``None`` is returned now. +- :gh:`2953`, [SunOS]: :meth:`Process.gids` returned a ``puids`` namedtuple + instead of ``pgids``. :meth:`Process.nice` was offset by +20 compared to + ``getpriority(3)``. :meth:`Process.net_connections` returned UNIX sockets + with ``type=-1`` and ``fd=-1``. Also, :meth:`Process.net_connections` now + raises :exc:`ZombieProcess` instead of ``RuntimeError`` for zombie processes. + +7.2.2 — 2026-01-28 +^^^^^^^^^^^^^^^^^^ + +**Performance** + +- :gh:`2705`, [Linux], [macOS], [BSD]: :meth:`Process.wait` no longer uses a + busy loop. It now uses ``pidfd_open()`` + ``poll()`` on Linux (requires Linux + >= 5.3 and Python >= 3.9), and ``kqueue()`` on macOS and BSD. + +**Bug fixes** + +- :gh:`2701`, [macOS], :label:`build-fail`: fix compilation error on macOS < + 10.7. (patch by :user:`Sergey Fedorov `) +- :gh:`2707`, [macOS], :label:`memleak`: fix potential memory leaks in error + paths of :meth:`Process.memory_full_info` and :meth:`Process.threads`. +- :gh:`2708`, [macOS]: :meth:`Process.cmdline` and :meth:`Process.environ` may + fail with ``OSError: [Errno 0] Undefined error`` (from + ``sysctl(KERN_PROCARGS2)``). They now raise :exc:`AccessDenied` instead. + +7.2.1 — 2025-12-29 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`2699`, [FreeBSD], [NetBSD]: :func:`heap_info` does not detect small + allocations (<= 1K). In order to fix that, we now flush internal jemalloc + cache before fetching the metrics. + +7.2.0 — 2025-12-23 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`1275`: new :func:`heap_info` and :func:`heap_trim` functions, providing + direct access to the platform's native C :term:`heap` allocator (glibc, + mimalloc, libmalloc). Useful to create tools to detect memory leaks. + +**Build and packaging** + +- :gh:`2680`, :label:`breaking`: unit tests are no longer installed / part of + the distribution. They now live under ``tests/`` instead of ``psutil/tests``, + so ``import psutil.tests`` no longer works (it was never documented to begin + with). +- :gh:`2403`, [Linux]: publish wheels for Linux musl. + +**Bug fixes** + +- :gh:`2684`, [FreeBSD], :label:`build-fail`: compilation fails on FreeBSD 14 + due to missing include. +- :gh:`2691`, [Windows], :label:`memleak`: fix memory leak in + :func:`net_if_stats` due to missing ``Py_CLEAR``. + +7.1.3 — 2025-11-02 +^^^^^^^^^^^^^^^^^^ + +**Internals** + +- :gh:`2667`: enforce ``clang-format`` on all C and header files. It is now the + mandatory formatting style for all C sources. +- :gh:`2676`, :gh:`2678`: replace unsafe ``sprintf`` / ``snprintf`` / + ``sprintf_s`` with ``str_format``, and ``strlcat`` / ``strlcpy`` with + ``str_copy`` / ``str_append``. Unifies string handling across platforms. + +**Bug fixes** + +- :gh:`2677`, [Windows], :label:`critical`: fix MAC address string construction + in :func:`net_if_addrs` (buffer overflow / misformat risk). +- :gh:`2679`, [OpenBSD], [NetBSD], :label:`build-fail`: can't build due to C + syntax error. +- :gh:`2672`, [macOS], [BSD]: increase the chances to recognize zombie + processes and raise the appropriate exception (:exc:`ZombieProcess`). +- :gh:`2674`, [Windows]: :func:`disk_usage` could truncate values on 32-bit + platforms, potentially reporting incorrect :field:`total`, :field:`free`, + :field:`used` space for drives larger than 4GB. +- :gh:`2675`, [macOS]: :meth:`Process.status` incorrectly returns + :data:`STATUS_RUNNING` for 99% of the processes. + +7.1.2 — 2025-10-25 +^^^^^^^^^^^^^^^^^^ + +**Build and packaging** + +- :gh:`2657`, :label:`breaking`: stop publishing prebuilt Linux and Windows + wheels for 32-bit Python. 32-bit CPython is still supported, but psutil must + now be built from source. +- :gh:`2565`: produce wheels for free-thread cPython 3.13 and 3.14. (patch by + :user:`Lysandros Nikolaou `) + +**Bug fixes** + +- :gh:`2658`, [macOS], :label:`critical`: double ``free()`` in + :meth:`Process.environ` when it fails internally. This posed a risk of + segfault. +- :gh:`2662`, [macOS], :label:`critical`: massive C code cleanup to guard + against possible segfaults which were (not so) sporadically spotted on CI. +- :gh:`2650`, [macOS]: :meth:`Process.cmdline` and :meth:`Process.environ` may + incorrectly raise :exc:`NoSuchProcess` instead of :exc:`ZombieProcess`. + +7.1.1 — 2025-10-19 +^^^^^^^^^^^^^^^^^^ + +**Internals** + +- :gh:`2646`, [SunOS]: add CI test runner for SunOS. + +**Dropped support** + +- :gh:`2645`, [SunOS], :label:`breaking`: dropped support for SunOS 10. + +**Bug fixes** + +- :gh:`2641`, [SunOS], :label:`build-fail`: cannot compile psutil from sources + due to missing C include. +- :gh:`2357`, [SunOS]: :meth:`Process.cmdline` does not handle spaces properly. + (patch by :user:`Ben Raz `) + +7.1.0 — 2025-09-17 +^^^^^^^^^^^^^^^^^^ + +**Build and packaging** + +- :gh:`2581`, [Windows]: publish ARM64 wheels. (patch by + :user:`Matthieu Darbois `) + +**Internals** + +- :gh:`2575`: introduced ``dprint`` CLI tool to format .yml and .md files. + +**Dropped support** + +- :gh:`2571`, [FreeBSD], :label:`breaking`: Dropped support for FreeBSD 8 and + earlier. FreeBSD 8 was maintained from 2009 to 2013. + +**Bug fixes** + +- :gh:`2545`, [SunOS], :label:`critical`: Fix handling of ``MIB2_UDP_ENTRY`` in + :func:`net_connections`. +- :gh:`2586`, [macOS], :label:`critical`: fixed different places in C code + which can trigger a segfault. +- :gh:`2610`, [macOS], :label:`critical`: fix :func:`cpu_freq` segfault on ARM + architectures. +- :gh:`2473`, [macOS], :label:`build-fail`: Fix build issue on macOS 11 and + lower. +- :gh:`2494`, [Windows]: :meth:`Process.memory_maps`, :meth:`Process.exe` and + :meth:`Process.open_files` now properly handle UNC paths (e.g. + ``\\??\\C:\\Windows\\Temp`` → ``C:\\Windows\\Temp``). (patch by + :user:`Ben Peddell `) +- :gh:`2506`, [Windows]: Windows service APIs had issues with unicode services + using special characters in their name. +- :gh:`2514`, [Linux]: :meth:`Process.cwd` sometimes fail with + :exc:`FileNotFoundError` due to a race condition. +- :gh:`2526`, [Linux]: :meth:`Process.create_time` now uses a monotonic clock, + preventing :meth:`Process.is_running` from returning wrong results after + system clock updates. (patch by :user:`Jonathan Kohler `) +- :gh:`2528`, [Linux]: :meth:`Process.children` may raise + :exc:`PermissionError`. It will now raise :exc:`AccessDenied` instead. +- :gh:`2540`, [macOS]: :func:`boot_time` is off by 45 seconds (C precision + issue). +- :gh:`2541`, :gh:`2570`, :gh:`2578`, [Linux], [macOS], [NetBSD]: + :meth:`Process.create_time` does not reflect system clock updates. +- :gh:`2542`: if system clock is updated :meth:`Process.children` and + :meth:`Process.parent` may not be able to return the right information. +- :gh:`2552`, [Windows]: :func:`boot_time` didn't take into account the time + spent during suspend / hibernation. +- :gh:`2560`, [Linux]: :meth:`Process.memory_maps` may crash with + :exc:`IndexError` on RISCV64 due to a malformed :proc:`/proc/pid/smaps` file. + (patch by :user:`Julien Stephan `) +- :gh:`2604`, [Linux]: :func:`virtual_memory` :field:`used` field does not + match recent versions of ``free`` CLI utility. (patch by + :user:`Isaac K. Ko <1saac-k>`) +- :gh:`2605`, [Linux]: :func:`sensors_battery` reports a negative amount for + seconds left. +- :gh:`2607`, [Windows]: :meth:`WindowsService.description` method may fail + with ``ERROR_NOT_FOUND``. Now it returns an empty string instead. + +7.0.0 — 2025-02-13 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`669`, [Windows]: :func:`net_if_addrs` also returns the + :field:`broadcast` address instead of ``None``. + +**API changes** + +- :gh:`2490`, :label:`breaking`: remove long deprecated + ``Process.memory_info_ex()`` (deprecated since 4.0.0). Use + :meth:`Process.memory_full_info` instead. + +**Dropped support** + +- :gh:`2480`, :label:`breaking`: drop Python 2.7 support. Latest version + supporting it is psutil 6.1.X (``pip2 install psutil==6.1.*``). + +**Bug fixes** + +- :gh:`2496`, [Linux], :label:`critical`: Avoid segfault (a cPython bug) on + :meth:`Process.memory_maps` for processes that use hundreds of GBs of memory. +- :gh:`2502`, [macOS]: :func:`virtual_memory` now uses ``host_statistics64`` + (same as ``vm_stat``), more accurate. + +6.1.1 — 2024-12-19 +^^^^^^^^^^^^^^^^^^ + +**Internals** + +- :gh:`2471`: use Vulture CLI tool to detect dead code. + +**Bug fixes** + +- :gh:`2418`, [Linux]: fix race condition in case :proc:`/proc/pid/stat` does + not exist, but :proc:`/proc/pid` does, resulting in :exc:`FileNotFoundError`. +- :gh:`2470`, [Linux]: :func:`users` may return "localhost" instead of the + actual IP address of the user logged in. + +6.1.0 — 2024-10-17 +^^^^^^^^^^^^^^^^^^ + +**Performance** + +- :gh:`2366`, [Windows]: drastically speedup :func:`process_iter` by using + process "fast" create time to determine process identity. +- :gh:`2457`, [AIX]: significantly improve the speed of + :meth:`Process.open_files` for some edge cases. + +**Internals** + +- :gh:`2446`: use pytest instead of unittest. +- :gh:`2448`: add ``make install-sysdeps`` target to install the necessary + system dependencies (python-dev, gcc, etc.) on all supported UNIX flavors. +- :gh:`2449`: add ``make install-pydeps-test`` and ``make install-pydeps-dev`` + targets. They can be used to install dependencies meant for running tests and + for local development. They can also be installed via ``pip install .[test]`` + and ``pip install .[dev]``. +- :gh:`2456`: allow running tests via ``python3 -m psutil.tests`` even if + ``pytest`` is not installed. + +**Bug fixes** + +- :gh:`2427`, :label:`critical`: psutil (segfault) on import in the + free-threaded (no GIL) version of Python 3.13. (patch by + :user:`Sam Gross `) +- :gh:`2455`, [Linux]: :exc:`IndexError` may occur when reading + :proc:`/proc/pid/stat` and field 40 (``blkio_ticks``) is missing. +- :gh:`2460`, [OpenBSD]: :meth:`Process.num_fds` and :meth:`Process.open_files` + may fail with :exc:`NoSuchProcess` for PID 0. Instead, we now return "null" + values (``0`` and ``[]`` respectively). + +6.0.0 — 2024-06-18 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`2396`: a new ``process_iter.cache_clear()`` API can be used to clear + :func:`process_iter` internal cache. + +**API changes** + +- :gh:`2109`, :label:`breaking`: the named tuple returned by + :func:`disk_partitions` no longer has the :field:`maxfile` and + :field:`maxpath` fields (they can be very slow to retrieve on NFS). +- :gh:`2407`, :label:`breaking`: rename :meth:`Process.connections` to + :meth:`Process.net_connections`. Old name still works but is deprecated. + +**Performance** + +- :gh:`2396`, :label:`breaking`: :func:`process_iter` no longer preemptively + checks whether PIDs have been reused, making it around **20x faster**. Use + :meth:`Process.is_running` on yielded instances instead (it also removes + reused PIDs from the internal cache). + +**Build and packaging** + +- :gh:`2369`, [Windows], :label:`build-fail`: wheels failed to build. Fixed by + upgrading ``cibuildwheel``. +- :gh:`2375`, [macOS]: provide arm64 wheels. (patch by + :user:`Matthieu Darbois `) +- :gh:`2401`, Support building with free-threaded CPython 3.13. (patch by + :user:`Sam Gross `) +- :gh:`2425`, [Linux]: provide aarch64 wheels. (patch by + :user:`Matthieu Darbois ` / :user:`Ben Raz `) + +**Internals** + +- :gh:`2366`, [Windows]: log debug message when using slower process APIs. + +**Bug fixes** + +- :gh:`2360`, [macOS], :label:`build-fail`: can't compile on macOS < 10.13. + (patch by :user:`Ryan Schmidt `) +- :gh:`2362`, [macOS], :label:`build-fail`: can't compile on macOS 10.11. + (patch by :user:`Ryan Schmidt `) +- :gh:`2365`, [macOS], :label:`build-fail`: can't compile on macOS < 10.9. + (patch by :user:`Ryan Schmidt `) +- :gh:`2412`, [macOS], :label:`build-fail`: can't compile on macOS 10.4 PowerPC + due to missing ``MNT_`` constants. +- :gh:`2250`, [NetBSD]: :meth:`Process.cmdline` sometimes fail with ``EBUSY`` + for long cmdlines. Now retries up to 50 times, returning an empty list as + last resort. +- :gh:`2254`, [Linux]: offline cpus raise :exc:`NotImplementedError` in + :func:`cpu_freq` (patch by :user:`Shade Gladden `) +- :gh:`2272`: Add pickle support to psutil Exceptions. +- :gh:`2359`, [Windows]: :func:`pid_exists` disagrees with :class:`Process` on + whether a pid exists when ``ERROR_ACCESS_DENIED``. +- :gh:`2395`, [OpenBSD]: :func:`pid_exists` erroneously return True if the + argument is a thread ID (TID) instead of a PID (process ID). + +5.9.8 — 2024-01-19 +^^^^^^^^^^^^^^^^^^ + +**Performance** + +- :gh:`2343`, [FreeBSD]: filter :func:`net_connections` in C instead of Python, + **~4x faster**. Only requested connection types are now retrieved. +- :gh:`2342`, [NetBSD]: same as above (:gh:`2343`) but for NetBSD. + +**Internals** + +- :gh:`2349`: adopted black formatting style. + +**Bug fixes** + +- :gh:`930`, [NetBSD], :label:`critical`, :label:`memleak`: + :func:`net_connections` implementation was broken. It could either leak + memory or core dump. +- :gh:`2345`, [Linux], :label:`build-fail`: fix compilation on older compiler + missing :data:`NIC_DUPLEX_UNKNOWN`. +- :gh:`2340`, [NetBSD]: if process is terminated, :meth:`Process.cwd` will + return an empty string instead of raising :exc:`NoSuchProcess`. +- :gh:`2222`, [macOS]: :func:`cpu_freq` now returns fixed values for + :field:`min` and :field:`max` frequencies in all Apple Silicon chips. + +5.9.7 — 2023-12-17 +^^^^^^^^^^^^^^^^^^ + +**Internals** + +- :gh:`2324`: enforce Ruff rule ``raw-string-in-exception``, which helps + providing clearer tracebacks when exceptions are raised by psutil. + +**Bug fixes** + +- :gh:`2321`, [Linux], [SunOS], :label:`build-fail`: missing + ``#include `` causes an implicit declaration of ``close()`` and + ``syscall()``. +- :gh:`2325`, [PyPy], :label:`build-fail`: psutil did not compile on PyPy due + to missing ``PyErr_SetExcFromWindowsErrWithFilenameObject`` cPython API. + +5.9.6 — 2023-10-15 +^^^^^^^^^^^^^^^^^^ + +**API changes** + +- :gh:`1703`: :func:`cpu_percent` and :func:`cpu_times_percent` are now thread + safe. +- :gh:`2266`: if :class:`Process` class is passed a very high PID, raise + :exc:`NoSuchProcess` instead of :exc:`OverflowError`. (patch by + :user:`Xuehai Pan `) +- :gh:`2290`: PID reuse is now preemptively checked for :meth:`Process.ppid` + and :meth:`Process.parents`. + +**Internals** + +- :gh:`2312`: use ``ruff`` linter instead of ``flake8 + isort``. + +**Dropped support** + +- :gh:`2246`, :label:`breaking`: drop python 3.4 & 3.5 support. (patch by + :user:`Matthieu Darbois `) + +**Bug fixes** + +- :gh:`2241`, [NetBSD], :label:`build-fail`: can't compile On NetBSD + 10.99.3/amd64. (patch by :user:`Thomas Klausner <0-wiz-0>`) +- :gh:`2195`, [Linux]: no longer print exception at import time in case + :proc:`/proc/stat` can't be read due to permission error. Redirect it to + :envvar:`PSUTIL_DEBUG` instead. +- :gh:`2245`, [Windows]: fix var unbound error on possibly in + :func:`swap_memory` (patch by :user:`student_2333 `) +- :gh:`2268`: ``bytes2human()`` utility function was unable to properly + represent negative values. +- :gh:`2252`, [Windows]: :func:`disk_usage` fails on Python 3.12+. (patch by + Matthieu Darbois) +- :gh:`2284`, [Linux]: :meth:`Process.memory_full_info` may incorrectly raise + :exc:`ZombieProcess` if it's determined via ``/proc/pid/smaps_rollup``. + Instead we now fallback on reading :proc:`/proc/pid/smaps`. +- :gh:`2287`, [OpenBSD], [NetBSD]: :meth:`Process.is_running` erroneously + return ``False`` for zombie processes, because creation time cannot be + determined. +- :gh:`2288`, [Linux]: correctly raise :exc:`ZombieProcess` on + :meth:`Process.exe`, :meth:`Process.cmdline` and :meth:`Process.memory_maps` + instead of returning a "null" value. +- :gh:`2290`: differently from what stated in the doc, PID reuse is not + preemptively checked for :meth:`Process.nice` (set), :meth:`Process.ionice`, + (set), :meth:`Process.cpu_affinity` (set), :meth:`Process.rlimit` (set), + :meth:`Process.parent`. +- :gh:`2308`, [OpenBSD]: :meth:`Process.threads` always fail with + :exc:`AccessDenied` (also as root). + +5.9.5 — 2023-04-17 +^^^^^^^^^^^^^^^^^^ + +**API changes** + +- :gh:`2230`, [OpenBSD]: :func:`net_connections` rewritten from scratch: now + retrieves :data:`socket.AF_UNIX` socket paths, is faster, and no longer + produces duplicates. +- :gh:`2238`: :meth:`Process.cwd` now consistently returns ``""`` on all + platforms when the directory can't be determined. +- :gh:`2239`, [UNIX]: for zombie processes, return the truncated + :meth:`Process.name` (15 chars) instead of raising :exc:`ZombieProcess` when + the full name can't be determined from :meth:`Process.cmdline`. + +**Internals** + +- :gh:`2196`: in case of exception, display a cleaner error traceback by hiding + the :exc:`KeyError` bit deriving from a missed cache hit. +- :gh:`2217`: print the full traceback when a :exc:`DeprecationWarning` or + :exc:`UserWarning` is raised. +- :gh:`2240`, [NetBSD], [OpenBSD]: add CI testing on every commit for NetBSD + and OpenBSD platforms (python 3 only). + +**Bug fixes** + +- :gh:`2164`, [Linux], :label:`build-fail`: compilation fails on kernels < + 2.6.27 (e.g. CentOS 5). +- :gh:`2186`, [FreeBSD], :label:`build-fail`: compilation fails with Clang 15. + (patch by :user:`Po-Chuan Hsieh `) +- :gh:`1043`, [OpenBSD] :func:`net_connections` returns duplicate entries. +- :gh:`1915`, [Linux]: on certain kernels, ``"MemAvailable"`` field from + :proc:`/proc/meminfo` returns ``0`` (possibly a kernel bug), in which case we + calculate an approximation for :field:`available` memory which matches "free" + CLI utility. +- :gh:`2191`, [Linux]: :func:`disk_partitions`: do not unnecessarily read + :proc:`/proc/filesystems` and raise :exc:`AccessDenied` unless user specified + ``all=False`` argument. +- :gh:`2216`, [Windows]: fix tests when running in a virtual environment (patch + by Matthieu Darbois) +- :gh:`2225`, [POSIX]: :func:`users` loses precision for :field:`started` field + (off by 1 minute). +- :gh:`2229`, [OpenBSD]: unable to properly recognize zombie processes. + :exc:`NoSuchProcess` may be raised instead of :exc:`ZombieProcess`. +- :gh:`2231`, [NetBSD]: :field:`available` :func:`virtual_memory` is higher + than :field:`total`. +- :gh:`2234`, [NetBSD]: :func:`virtual_memory` metrics are wrong: + :field:`available` and :field:`used` are too high. We now match values shown + by *htop* CLI utility. +- :gh:`2236`, [NetBSD]: :meth:`Process.num_threads` and :meth:`Process.threads` + return threads that are already terminated. +- :gh:`2237`, [OpenBSD], [NetBSD]: :meth:`Process.cwd` may raise + :exc:`FileNotFoundError` if cwd no longer exists. Return an empty string + instead. + +5.9.4 — 2022-11-07 +^^^^^^^^^^^^^^^^^^ + +**Build and packaging** + +- :gh:`2102`: use Limited API when building wheels with CPython 3.6+ on Linux, + macOS and Windows. (patch by :user:`Matthieu Darbois `) + +**Bug fixes** + +- :gh:`2156`, [Linux], :label:`build-fail`: compilation may fail on very old + gcc compilers due to missing ``SPEED_UNKNOWN`` definition. (patch by + :user:`Amir Rossert `) +- :gh:`2010`, [macOS], :label:`build-fail`: on MacOS, arm64 ``IFM_1000_TX`` and + ``IFM_1000_T`` are the same value, causing a build failure. (patch by + :user:`Lawrence D'Anna `) +- :gh:`2077`, [Windows]: Use system-level values for :func:`virtual_memory`. + (patch by :user:`Daniel Widdis `) +- :gh:`2160`, [Windows]: get :func:`swap_memory` :field:`percent` usage from + performance counters. (patch by :user:`Daniel Widdis `) + +5.9.3 — 2022-10-18 +^^^^^^^^^^^^^^^^^^ + +**Build and packaging** + +- :gh:`2040`, [macOS]: provide wheels for arm64 architecture. (patch by + Matthieu Darbois) + +**Internals** + +- :gh:`2153`, [macOS] Fix race condition in + ``test_posix.TestProcess.test_cmdline``. (patch by + :user:`Matthieu Darbois `) + +**Bug fixes** + +- :gh:`2135`, [macOS], :label:`critical`: :meth:`Process.environ` may contain + garbage data. Fix out-of-bounds read around ``sysctl_procargs``. (patch by + :user:`Bernhard Urban-Forster `) +- :gh:`2138`, [Linux], :label:`build-fail`: can't compile psutil on Android due + to undefined ``ethtool_cmd_speed`` symbol. +- :gh:`2116`, [macOS]: :func:`net_connections` fails with :exc:`RuntimeError`. +- :gh:`2142`, [POSIX]: :func:`net_if_stats` 's :field:`flags` on Python 2 + returned unicode instead of str. (patch by :user:`Matthieu Darbois `) +- :gh:`2147`, [macOS] Fix disk usage report on macOS 12+. (patch by + :user:`Matthieu Darbois `) +- :gh:`2150`, [Linux] :meth:`Process.threads` may raise :exc:`NoSuchProcess`. + Fix race condition. (patch by :user:`Daniel Li `) + +5.9.2 — 2022-09-04 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`2093`, [FreeBSD]: :func:`pids` may fail with ENOMEM. Dynamically + increase the ``malloc()`` buffer size until it's big enough. +- :gh:`2095`, [Linux]: :func:`net_if_stats` returns incorrect interface speed + for 100GbE network cards. +- :gh:`2113`, [FreeBSD]: :func:`virtual_memory` may raise ENOMEM due to missing + ``#include `` directive. (patch by + :user:`Peter Jeremy `) +- :gh:`2128`, [NetBSD]: :func:`swap_memory` was miscalculated. (patch by + :user:`Thomas Klausner <0-wiz-0>`) + +5.9.1 — 2022-05-20 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`2037`: add :field:`flags` field to :func:`net_if_stats`. +- :gh:`2057`, [OpenBSD]: add support for :func:`cpu_freq`. + +**Performance** + +- :gh:`2050`, [Linux]: increase :manpage:`read(2)` buffer size from 1k to 32k + when reading ``/proc`` pseudo files line by line. +- :gh:`2107`, [Linux]: :meth:`Process.memory_full_info` now reads + ``/proc/pid/smaps_rollup`` instead of :proc:`/proc/pid/smaps` + (**5x faster**). + +**Dropped support** + +- :gh:`1053`, :label:`breaking`: drop Python 2.6 support. (patches by Matthieu + Darbois and Hugo van Kemenade) + +**Bug fixes** + +- :gh:`2048`: :exc:`AttributeError` is raised if :exc:`psutil.Error` class is + raised manually and passed through ``str``. +- :gh:`2049`, [Linux]: :func:`cpu_freq` erroneously returns :field:`current` + value in GHz while :field:`min` and :field:`max` are in MHz. +- :gh:`2050`, [Linux]: :func:`virtual_memory` may raise :exc:`ValueError` if + running in a LCX container. + +5.9.0 — 2021-12-29 +^^^^^^^^^^^^^^^^^^ + +**New platforms** + +- :gh:`1996`, [BSD]: add support for MidnightBSD. (patch by + :user:`Saeed Rasooli `) + +**API changes** + +- :gh:`1992`: :exc:`NoSuchProcess` message now specifies if the PID has been + reused. Error classes now have improved ``__repr__`` and ``__str__``. +- :gh:`1999`, [Linux]: :func:`disk_partitions`: convert ``/dev/root`` device + (an alias used on some Linux distros) to real root device path. + +**Performance** + +- :gh:`1851`, [Linux]: :func:`cpu_freq` reads from :proc:`/proc/cpuinfo` + instead of many files in ``/sys`` fs, faster on systems with many CPUs. + (patch by marxin) + +**Documentation** + +- :gh:`2042`: rewrite HISTORY.rst to use hyperlinks pointing to psutil API doc. + +**Internals** + +- :gh:`2005`: :envvar:`PSUTIL_DEBUG` mode now prints file name and line number + of the debug messages coming from C extension modules. + +**Bug fixes** + +- :gh:`1953`, [Windows], :label:`critical`: :func:`disk_partitions` crashes due + to insufficient buffer len. (patch by :user:`MaWe2019 `) +- :gh:`1965`, [Windows], :label:`critical`: fix "Fatal Python error: + deallocating None" when calling :func:`users` multiple times. +- :gh:`1990`, [Windows], :label:`memleak`: starting a :class:`WindowsService` + leaked a service handle. +- :gh:`1512`, [macOS]: sometimes :meth:`Process.connections` will crash with + ``EOPNOTSUPP`` for one connection; this is now ignored. +- :gh:`1598`, [Windows]: :func:`disk_partitions` only returns mount points on + drives where it first finds one. +- :gh:`1874`, [SunOS]: swap output error due to incorrect range. +- :gh:`1892`, [macOS]: :func:`cpu_freq` broken on Apple M1. Also, :field:`min` + and :field:`max` are set to 0 if they can't be determined, instead of raising + an exception. +- :gh:`1901`, [macOS]: :meth:`Process.open_files`, :meth:`Process.connections` + and others could randomly raise :exc:`AccessDenied` because the internal + buffer of ``proc_pidinfo(PROC_PIDLISTFDS)`` was too small. Now dynamically + increased until sufficient. +- :gh:`1904`, [Windows]: ``OpenProcess`` fails with ``ERROR_SUCCESS`` due to + ``GetLastError()`` called after ``sprintf()``. (patch by :user:`alxchk`) +- :gh:`1913`, [Linux]: :func:`wait_procs` should catch + :exc:`subprocess.TimeoutExpired` exception. +- :gh:`1919`, [Linux]: :func:`sensors_battery` can raise :exc:`TypeError` on + PureOS. +- :gh:`1921`, [Windows]: :func:`swap_memory` shows committed memory instead of + swap. +- :gh:`1940`, [Linux]: psutil does not handle ``ENAMETOOLONG`` when accessing + process file descriptors in procfs. (patch by + :user:`Nikita Radchenko `) +- :gh:`1948`: ``memoize_when_activated`` decorator is not thread-safe. (patch + by :user:`Xuehai Pan `) +- :gh:`1980`, [Windows]: 32bit / WoW64 processes fails to read + :meth:`Process.name` longer than 128 characters. (patch by + :user:`PetrPospisil `) +- :gh:`1991`: :func:`process_iter` is not thread safe and can raise + :exc:`TypeError` if invoked from multiple threads. +- :gh:`1956`, [macOS]: :meth:`Process.cpu_times` reports incorrect timings on + M1 machines. (patch by :user:`Olivier Dormond `) +- :gh:`2023`, [Linux]: :func:`cpu_freq` return order is wrong on systems with + more than 9 CPUs. + +5.8.0 — 2020-12-19 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`1863`: :func:`disk_partitions` exposes 2 extra fields: :field:`maxfile` + and :field:`maxpath`, which are the maximum file name and path name length. + +**New platforms** + +- :gh:`1872`, [Windows]: added support for PyPy 2.7. + +**Build and packaging** + +- :gh:`1879`: provide pre-compiled wheels for Linux and macOS (yey!). + +**Internals** + +- :gh:`1880`: switch CI from Travis/Cirrus to GitHub Actions (Linux, macOS, + FreeBSD). AppVeyor still used for Windows. + +**Bug fixes** + +- :gh:`1708`, [Linux]: get rid of :func:`sensors_temperatures` duplicates. + (patch by :user:`Tim Schlueter `). +- :gh:`1839`, [Windows]: always raise :exc:`AccessDenied` instead of + :exc:`WindowsError` when failing to query 64 processes from 32 bit ones by + using ``NtWoW64`` APIs. +- :gh:`1866`, [Windows]: :meth:`Process.exe`, :meth:`Process.cmdline`, + :meth:`Process.environ` may raise "[WinError 998] Invalid access to memory + location" on Python 3.9 / VS 2019. +- :gh:`1874`, [SunOS]: wrong swap output given when encrypted column is + present. +- :gh:`1875`, [Windows]: :meth:`Process.username` may raise + ``ERROR_NONE_MAPPED`` if the SID has no corresponding account name. In this + case :exc:`AccessDenied` is now raised. +- :gh:`1886`, [macOS]: ``EIO`` error may be raised on :meth:`Process.cmdline` + and :meth:`Process.environ`. Now it gets translated into :exc:`AccessDenied`. +- :gh:`1887`, [Windows]: ``OpenProcess`` may fail with "[WinError 0] The + operation completed successfully"." Turn it into :exc:`AccessDenied` or + :exc:`NoSuchProcess` depending on whether the PID is alive. +- :gh:`1891`, [macOS]: get rid of deprecated ``getpagesize()``. + +5.7.3 — 2020-10-24 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`809`, [FreeBSD]: add support for :meth:`Process.rlimit`. +- :gh:`893`, [BSD]: add support for :meth:`Process.environ` (patch by + :user:`Armin Gruner `) + +**API changes** + +- :gh:`1830`, [POSIX]: :func:`net_if_stats` :field:`isup` also checks whether + the :term:`NIC` is running (meaning Wi-Fi or ethernet cable is connected). + (patch by Chris Burger) +- :gh:`1837`, [Linux]: improved battery detection and charge :field:`secsleft` + calculation (patch by :user:`aristocratos`) + +**Bug fixes** + +- :gh:`1823`, [Windows], :label:`critical`: :meth:`Process.open_files` may + cause a segfault due to a NULL pointer. +- :gh:`1791`, [macOS], :label:`build-fail`: fix missing include for + ``getpagesize()``. +- :gh:`1620`, [Linux]: :func:`cpu_count` with ``logical=False`` result is + incorrect on systems with more than one CPU socket. (patch by + :user:`Vincent A. Arcila `) +- :gh:`1738`, [macOS]: :meth:`Process.exe` may raise :exc:`FileNotFoundError` + if process is still alive but the exe file which launched it got deleted. +- :gh:`1838`, [Linux]: :func:`sensors_battery`: if :field:`percent` can be + determined but not the remaining values, still return a result instead of + ``None``. (patch by :user:`aristocratos`) + +5.7.2 — 2020-07-15 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- wheels for 2.7 were inadvertently deleted. + +5.7.1 — 2020-07-15 +^^^^^^^^^^^^^^^^^^ + +**New platforms** + +- :gh:`1768`, [Windows]: added support for Windows Nano Server. (contributed by + Julien Lebot) + +**API changes** + +- :gh:`1747`: :meth:`Process.wait` return value is cached so that the exit code + can be retrieved on then next call. +- :gh:`1747`, [POSIX]: :meth:`Process.wait` on POSIX now returns an enum, + showing the negative signal which was used to terminate the process. It + returns something like ````. +- :gh:`1747`: :class:`Process` class provides more info about the process on + ``str()`` and ``repr()`` (status and exit code). + +**Performance** + +- :gh:`1741`, [POSIX]: ``make build`` now runs in parallel on Python >= 3.6 and + it's about **15% faster**. + +**Internals** + +- :gh:`1709`: parallel tests on POSIX (``make test-parallel``). They're twice + as fast! +- :gh:`1757`: memory leak tests are now stable. + +**Bug fixes** + +- :gh:`1781`, :label:`critical`: :func:`getloadavg` can crash the Python + interpreter. (patch by :user:`Ammar Askar `) +- :gh:`1726`, [Linux]: :func:`cpu_freq` parsing should use spaces instead of + tabs on ia64. (patch by :user:`MichaÅ‚ Górny `) +- :gh:`1760`, [Linux]: :meth:`Process.rlimit` does not handle long long type + properly. +- :gh:`1766`, [macOS]: :exc:`NoSuchProcess` may be raised instead of + :exc:`ZombieProcess`. + +5.7.0 — 2020-02-18 +^^^^^^^^^^^^^^^^^^ + +**New platforms** + +- :gh:`1637`, [SunOS]: add partial support for old SunOS 5.10 Update 0 to 3. +- :gh:`1686`, [Windows]: added support for PyPy on Windows. + +**API changes** + +- :gh:`1648`, [Linux]: :func:`sensors_temperatures` looks into an additional + ``/sys/device/`` directory for additional data. (patch by + :user:`Javad Karabi `) +- :gh:`1677`, [Windows]: :meth:`Process.exe` will succeed for all process PIDs + (instead of raising :exc:`AccessDenied`). +- :gh:`1693`, [Windows]: :func:`boot_time`, :meth:`Process.create_time` and + :func:`users`'s login time now have 1 micro second precision (before the + precision was of 1 second). + +**Performance** + +- :gh:`1679`, [Windows]: :func:`net_connections` and + :meth:`Process.connections` are **10% faster**. + +**Internals** + +- :gh:`1671`, [FreeBSD]: add CI testing/service for FreeBSD (Cirrus CI). +- :gh:`1682`, [PyPy]: added CI / test integration for PyPy via Travis. + +**Dropped support** + +- :gh:`1652`, [Windows], :label:`breaking`: dropped support for Windows XP and + Windows Server + 2003. Minimum supported Windows version now is Windows Vista. + +**Bug fixes** + +- :gh:`1646`, [FreeBSD], :label:`critical`: many :class:`Process` methods may + cause a segfault due to a backward incompatible change in a C type on FreeBSD + 12.0. +- :gh:`1695`, [Linux], :label:`build-fail`: could not compile on kernels <= + 2.6.13 due to ``PSUTIL_HAS_IOPRIO`` not being defined. (patch by + :user:`Anselm Kruis `) +- :gh:`1538`, [NetBSD]: :meth:`Process.cwd` may return ``ENOENT`` instead of + :exc:`NoSuchProcess`. +- :gh:`1627`, [Linux]: :meth:`Process.memory_maps` can raise :exc:`KeyError`. +- :gh:`1642`, [SunOS]: querying basic info for PID 0 results in + :exc:`FileNotFoundError`. +- :gh:`1656`, [Windows]: :meth:`Process.memory_full_info` raises + :exc:`AccessDenied` even for the current user and :func:`os.getpid`. +- :gh:`1660`, [Windows]: :meth:`Process.open_files` rewritten with proper error + handling. +- :gh:`1662`, [Windows]: :meth:`Process.exe` may raise "[WinError 0] The + operation completed successfully". +- :gh:`1665`, [Linux]: :func:`disk_io_counters` does not take into account + extra fields added to recent kernels. (patch by + :user:`Mike Hommey `) +- :gh:`1672`: use the right C type when dealing with PIDs (int or long). +- :gh:`1673`, [OpenBSD]: :meth:`Process.connections`, :meth:`Process.num_fds` + and :meth:`Process.threads` raised wrong exception if process is gone. +- :gh:`1674`, [SunOS]: :func:`disk_partitions` may raise :exc:`OSError`. +- :gh:`1684`, [Linux]: :func:`disk_io_counters` may raise :exc:`ValueError` on + systems not having :proc:`/proc/diskstats`. + +5.6.7 — 2019-11-26 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`1630`, [Windows], :label:`build-fail`: can't compile source distribution + due to C syntax error. + +5.6.6 — 2019-11-25 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`1616`, :label:`critical`: use of ``Py_DECREF`` instead of ``Py_CLEAR`` will + result in double ``free()`` and segfault (`CVE-2019-18874 + `__). (patch + by Riccardo Schirone) +- :gh:`1619`, [OpenBSD], :label:`build-fail`: compilation fails due to C syntax + error. (patch by :user:`Nathan Houghton `) +- :gh:`1179`, [Linux]: :meth:`Process.cmdline` now handles processes that use + inappropriate chars to separate args. + +5.6.5 — 2019-11-06 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`1615`, :label:`build-fail`: remove ``pyproject.toml`` as it was causing + installation issues. + +5.6.4 — 2019-11-04 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`1527`, [Linux]: added :meth:`Process.cpu_times` :field:`iowait` counter, + which is the time spent waiting for blocking I/O to complete. + +**Build and packaging** + +- :gh:`1565`: add PEP 517/8 build backend and requirements specification for + better pip integration. (patch by :user:`Bernát Gábor `) + +**Bug fixes** + +- :gh:`1126`, [Linux], :label:`critical`: :meth:`Process.cpu_affinity` + segfaults on CentOS 5 / manylinux. :meth:`Process.cpu_affinity` support for + CentOS 5 was removed. +- :gh:`1528`, [AIX], :label:`build-fail`: compilation error on AIX 7.2 due to + 32 vs 64 bit differences. (patch by :user:`Arnon Yaari `) +- :gh:`1606`, [SunOS], :label:`build-fail`: compilation fails on SunOS 5.10. + (patch by vser1) +- :gh:`875`, [Windows]: :meth:`Process.cmdline`, :meth:`Process.environ` or + :meth:`Process.cwd` may occasionally fail with ``ERROR_PARTIAL_COPY`` which + now gets translated to :exc:`AccessDenied`. +- :gh:`1535`: :field:`type` and :field:`family` fields returned by + :func:`net_connections` are not always turned into enums. +- :gh:`1536`, [NetBSD]: :meth:`Process.cmdline` erroneously raise + :exc:`ZombieProcess` error if cmdline has non encodable chars. +- :gh:`1546`: usage percent may be rounded to 0 on Python 2. +- :gh:`1552`, [Windows]: :func:`getloadavg` math for calculating 5 and 15 mins + values is incorrect. +- :gh:`1568`, [Linux]: use CC compiler env var if defined. +- :gh:`1570`, [Windows]: ``NtWow64*`` syscalls fail to raise the proper error + code +- :gh:`1585`, [macOS]: avoid calling ``close()`` (in C) on possible negative + integers. (patch by :user:`Athos Ribeiro `) + +5.6.3 — 2019-06-11 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`1494`, [AIX]: added support for :meth:`Process.environ`. (patch by + :user:`Arnon Yaari `) + +**Bug fixes** + +- :gh:`1276`, [AIX]: can't get whole :meth:`Process.cmdline`. (patch by + :user:`Arnon Yaari `) +- :gh:`1501`, [Windows]: :meth:`Process.cmdline` and :meth:`Process.exe` raise + unhandled "WinError 1168 element not found" exceptions for "Registry" and + "Memory Compression" pseudo processes on Windows 10. +- :gh:`1526`, [NetBSD]: :meth:`Process.cmdline` could raise :exc:`MemoryError`. + (patch by :user:`Kamil Rytarowski `) + +5.6.2 — 2019-04-26 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`604`, [Windows]: add new :func:`getloadavg`, returning system load + average calculation, including on Windows (emulated). (patch by + :user:`Ammar Askar `) +- :gh:`1476`, [Windows]: :meth:`Process.ionice` can now set high I/O priority. + New constants: :data:`IOPRIO_VERYLOW`, :data:`IOPRIO_LOW`, + :data:`IOPRIO_NORMAL`, :data:`IOPRIO_HIGH`. + +**API changes** + +- :gh:`1404`, [Linux]: :func:`cpu_count` with ``logical=False`` falls back to + reading ``/sys/devices/system/cpu/*/topology/core_id`` if + :proc:`/proc/cpuinfo` doesn't provide the info. + +**Documentation** + +- :gh:`1464`: various docfixes (always point to Python 3 doc, fix links, etc.). + +**Internals** + +- :gh:`1458`: provide coloured test output. Also show failures on + ``KeyboardInterrupt``. +- :gh:`1478`: add make command to re-run tests failed on last run. +- :gh:`1462`, [Linux]: (tests) make tests invariant to ``LANG`` setting (patch + by Benjamin Drung) +- :gh:`1463`: :src:`scripts/cpu_distribution.py` was broken. + +**Bug fixes** + +- :gh:`1480`, [Windows], :label:`critical`: :func:`cpu_count` with + ``logical=False`` could cause a crash due to fixed read violation. (patch by + Samer Masterson) +- :gh:`1491`, [SunOS], :label:`memleak`: :func:`net_if_addrs`: use ``free()`` + against ``ifap`` struct on error. (patch by Agnewee) +- :gh:`1223`, [Windows]: :func:`boot_time` may return incorrect value on + Windows XP. +- :gh:`1456`, [Linux]: :func:`cpu_freq` returns ``None`` instead of 0.0 when + :field:`min` and :field:`max` fields can't be determined. (patch by + :user:`Alex Manuskin `) +- :gh:`1470`, [Linux]: :func:`disk_partitions`: fix corner case when + ``/etc/mtab`` doesn't exist. (patch by + :user:`Cedric Lamoriniere `) +- :gh:`1471`, [SunOS]: :meth:`Process.name` and :meth:`Process.cmdline` can + return :exc:`SystemError`. (patch by :user:`Daniel Beer `) +- :gh:`1472`, [Linux]: :func:`cpu_freq` does not return all CPUs on + Raspberry-pi 3. +- :gh:`1474`: fix formatting of ``psutil.tests()`` which mimics ``ps aux`` + output. +- :gh:`1475`, [Windows]: :attr:`OSError.winerror` attribute wasn't properly + checked resulting in ``WindowsError(ERROR_ACCESS_DENIED)`` being raised + instead of :exc:`AccessDenied`. +- :gh:`1477`, [Windows]: wrong or absent error handling for private + ``NTSTATUS`` Windows APIs. Different process methods were affected by this. +- :gh:`1486`, [AIX], [SunOS]: :exc:`AttributeError` when interacting with + :class:`Process` methods involved into :meth:`Process.oneshot` context. +- :gh:`1493`, [Linux]: :func:`cpu_freq`: handle the case where + ``/sys/devices/system/cpu/cpufreq/`` exists but it's empty. + +5.6.1 — 2019-03-11 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`1448`, [Windows], :label:`critical`: crash on import due to + ``rtlIpv6AddressToStringA`` not available on Wine. +- :gh:`1451`, [Windows], :label:`critical`: :meth:`Process.memory_full_info` + segfaults. ``NtQueryVirtualMemory`` is now used instead of + ``QueryWorkingSet`` to calculate :term:`USS` memory. +- :gh:`1329`, [AIX], :label:`build-fail`: psutil doesn't compile on AIX 6.1. + (patch by :user:`Arnon Yaari `) + +5.6.0 — 2019-03-05 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`1433`: new :meth:`Process.parents` method. (idea by Ghislain Le Meur) + +**API changes** + +- :gh:`1291`, [macOS], :label:`critical`, :label:`breaking`: + :meth:`Process.memory_maps` was removed because inherently broken (segfault) + for years. +- :gh:`1420`, [Windows]: in case of exception :func:`disk_usage` now also shows + the path name. +- :gh:`1437`: :func:`pids` are returned in sorted order. + +**Performance** + +- :gh:`1379`, [Windows]: :meth:`Process.suspend` and :meth:`Process.resume` now + use ``NtSuspendProcess`` / ``NtResumeProcess`` instead of stopping / resuming + all threads. Faster and more reliable. +- :gh:`1422`, [Windows]: DLL-loaded Windows APIs are now loaded once on startup + instead of per function call, significantly faster. +- :gh:`1426`, [Windows]: ``PAGESIZE`` and number of processors is now + calculated on startup. + +**Internals** + +- :gh:`1428`: in case of error, the traceback message now shows the underlying + C function called which failed. +- :gh:`1442`: Python 3 is now the default interpreter used by Makefile. + +**Bug fixes** + +- :gh:`1411`, [BSD], :label:`critical`, :label:`memleak`: lack of ``Py_DECREF`` + could cause segmentation fault on process instantiation. +- :gh:`1353`: :func:`process_iter` is now thread safe (it rarely raised + :exc:`TypeError`). +- :gh:`1394`, [Windows]: :meth:`Process.name` and :meth:`Process.exe` may + erroneously return "Registry" or fail with "[Error 0] The operation completed + successfully". ``QueryFullProcessImageNameW`` is now used instead of + ``GetProcessImageFileNameW`` in order to prevent that. +- :gh:`1419`, [Windows]: :meth:`Process.environ` raises + :exc:`NotImplementedError` when querying a 64-bit process in 32-bit-WoW mode. + Now it raises :exc:`AccessDenied`. +- :gh:`1427`, [macOS]: :meth:`Process.cmdline` and :meth:`Process.environ` may + erroneously raise :exc:`OSError` on failed ``malloc()``. +- :gh:`1429`, [Windows]: ``SE DEBUG`` was not properly set for current process. + It is now, and it should result in less :exc:`AccessDenied` exceptions for + low PID processes. +- :gh:`1432`, [Windows]: ``Process.memory_info_ex()``'s USS memory is + miscalculated because we're not using the actual system ``PAGESIZE``. +- :gh:`1439`, [NetBSD]: :meth:`Process.connections` may return incomplete + results if using :meth:`Process.oneshot`. +- :gh:`1447`: original exception wasn't turned into :exc:`NoSuchProcess` / + :exc:`AccessDenied` exceptions when using :meth:`Process.oneshot` context + manager. + +5.5.1 — 2019-02-15 +^^^^^^^^^^^^^^^^^^ + +**API changes** + +- :gh:`1384`, [Windows]: on Windows >= 8.1, :meth:`Process.cmdline` falls back + to ``NtQueryInformationProcess`` on ``ERROR_ACCESS_DENIED``. (patch by + EccoTheFlintstone) + +**Bug fixes** + +- :gh:`1408`, [AIX], :label:`build-fail`: psutil won't compile on AIX 7.1 due + to missing header. (patch by :user:`Arnon Yaari `) +- :gh:`1394`, [Windows]: :meth:`Process.exe` returns "[Error 0] The operation + completed successfully" when Python process runs in "Virtual Secure Mode". +- :gh:`1402`: psutil exceptions' ``repr()`` show the internal private module + path. + +5.5.0 — 2019-01-23 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`1350`, [FreeBSD]: added support for :func:`sensors_temperatures`. (patch + by Alex Manuskin) +- :gh:`1352`, [FreeBSD]: added support for :func:`cpu_freq`. (patch by + :user:`Alex Manuskin `) + +**Bug fixes** + +- :gh:`1111`: :meth:`Process.oneshot` is now thread safe. +- :gh:`1354`, [Linux]: :func:`disk_io_counters` fails on Linux kernel 4.18+. +- :gh:`1357`, [Linux]: :meth:`Process.memory_maps` and + :meth:`Process.io_counters` methods are no longer exposed if not supported by + the kernel. +- :gh:`1368`, [Windows]: fix :meth:`Process.ionice` mismatch. (patch by + EccoTheFlintstone) +- :gh:`1370`, [Windows]: improper usage of ``CloseHandle()`` may lead to + override the original error code when raising an exception. +- :gh:`1373`: incorrect handling of cache in :meth:`Process.oneshot` context + causes :class:`Process` instances to return incorrect results. +- :gh:`1376`, [Windows]: ``OpenProcess`` now uses + ``PROCESS_QUERY_LIMITED_INFORMATION`` where possible, reducing + :exc:`AccessDenied` for system processes. +- :gh:`1376`, [Windows]: check if variable is ``NULL`` before ``free()`` ing + it. (patch by :user:`EccoTheFlintstone `) + +5.4.8 — 2018-10-30 +^^^^^^^^^^^^^^^^^^ + +**API changes** + +- :gh:`1197`, [Linux]: :func:`cpu_freq` falls back to :proc:`/proc/cpuinfo` if + ``/sys/devices/system/cpu/*`` is not available. +- :gh:`1310`, [Linux]: :func:`sensors_temperatures` falls back to + ``/sys/class/thermal`` if ``/sys/class/hwmon`` is not available (e.g. + Raspberry Pi). (patch by :user:`Alex Manuskin `) + +**Build and packaging** + +- :gh:`1320`, [POSIX]: better compilation support when using g++ instead of + GCC. (patch by :user:`Jaime Fullaondo `) + +**Bug fixes** + +- :gh:`715`: do not print exception on import time in case :func:`cpu_times` + fails. +- :gh:`1004`, [Linux]: :meth:`Process.io_counters` may raise :exc:`ValueError`. +- :gh:`1277`, [macOS]: available and used memory (:func:`virtual_memory`) + metrics are not accurate. +- :gh:`1294`, [Windows]: :meth:`Process.connections` may sometimes fail with + intermittent ``0xC0000001``. (patch by + :user:`Sylvain Duchesne `) +- :gh:`1307`, [Linux]: :func:`disk_partitions` does not honour + :data:`PROCFS_PATH`. +- :gh:`1320`, [AIX]: system CPU times (:func:`cpu_times`) were being reported + with ticks unit as opposed to seconds. (patch by + :user:`Jaime Fullaondo `) +- :gh:`1332`, [macOS]: psutil debug messages are erroneously printed all the + time. (patch by :user:`Ilya Yanok `) +- :gh:`1346`, [SunOS]: :func:`net_connections` returns an empty list. (patch by + Oleksii Shevchuk) + +5.4.7 — 2018-08-14 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`1309`, [Linux]: added :data:`STATUS_PARKED` constant for + :meth:`Process.status`. + +**API changes** + +- :gh:`1286`, [macOS]: :data:`OSX` constant is now deprecated in favor of new + :data:`MACOS`. +- :gh:`1321`, [Linux]: :func:`disk_io_counters` falls back to ``/sys/block`` if + :proc:`/proc/diskstats` is not available. (patch by + :user:`Lawrence Ye `) + +**Bug fixes** + +- :gh:`1209`, [macOS]: :meth:`Process.memory_maps` may fail with ``EINVAL`` due + to poor ``task_for_pid()`` syscall. :exc:`AccessDenied` is now raised + instead. +- :gh:`1278`, [macOS]: :meth:`Process.threads` incorrectly return microseconds + instead of seconds. (patch by :user:`Nikhil Marathe `) +- :gh:`1279`, [Linux], [macOS], [BSD]: :func:`net_if_stats` may return + ``ENODEV``. +- :gh:`1294`, [Windows]: :meth:`Process.connections` may sometime fail with + :exc:`MemoryError`. (patch by :user:`sylvainduchesne`) +- :gh:`1305`, [Linux]: :func:`disk_io_counters` may report inflated r/w bytes + values. +- :gh:`1309`, [Linux]: :meth:`Process.status` is unable to recognize + :field:`idle` and :field:`parked` statuses (returns ``"?"``). +- :gh:`1313`, [Linux]: :func:`disk_io_counters` can report inflated values due + to counting base disk device and its partition(s) twice. +- :gh:`1323`, [Linux]: :func:`sensors_temperatures` may fail with + :exc:`ValueError`. + +5.4.6 — 2018-06-07 +^^^^^^^^^^^^^^^^^^ + +**API changes** + +- :gh:`1273`: :func:`net_if_addrs` named tuple's name has been renamed from + ``snic`` to ``snicaddr``. + +**Bug fixes** + +- :gh:`1258`, [Windows], :label:`critical`: :meth:`Process.username` may cause + a segfault (Python interpreter crash). (patch by + :user:`Jean-Luc Migot `) +- :gh:`1274`, [Linux]: there was a small chance :meth:`Process.children` may + swallow :exc:`AccessDenied` exceptions. + +5.4.5 — 2018-04-13 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`1268`: setup.py's ``extra_require`` parameter requires latest setuptools + version, breaking quite a lot of installations. + +5.4.4 — 2018-04-13 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`1239`, [Linux]: expose kernel :field:`slab` memory field for + :func:`virtual_memory`. (patch by :user:`Maxime Mouial `) + +**API changes** + +- :gh:`771`, [Windows], :label:`breaking`: :func:`cpu_count` with + ``logical=False`` on Windows XP and Vista is no longer supported and returns + ``None``. + +**Bug fixes** + +- :gh:`1194`, [SunOS], :label:`critical`: fix double ``free()`` in + :meth:`Process.cpu_num`. (patch by Georg Sauthoff) +- :gh:`694`, [SunOS]: :meth:`Process.cmdline` could be truncated at the 15th + character when reading it from ``/proc``. An extra effort is made by reading + it from process address space first. (patch by + :user:`Georg Sauthoff `) +- :gh:`771`, [Windows]: :func:`cpu_count` (both logical and cores) return a + wrong (smaller) number on systems using process groups (> 64 cores). +- :gh:`771`, [Windows]: :func:`cpu_times` with ``percpu=True`` return fewer + CPUs on systems using process groups (> 64 cores). +- :gh:`771`, [Windows]: :func:`cpu_stats` and :func:`cpu_freq` may return + incorrect results on systems using process groups (> 64 cores). +- :gh:`1193`, [SunOS]: return uid/gid from ``/proc/pid/psinfo`` if there aren't + enough permissions for ``/proc/pid/cred``. (patch by + :user:`Georg Sauthoff `) +- :gh:`1194`, [SunOS]: return nice value from ``psinfo`` as ``getpriority()`` + doesn't support real-time processes. (patch by + :user:`Georg Sauthoff `) +- :gh:`1194`, [SunOS]: fix undefined behavior related to strict-aliasing rules + and warnings. (patch by :user:`Georg Sauthoff `) +- :gh:`1210`, [Linux]: :func:`cpu_percent` steal time may remain stuck at 100% + due to Linux erroneously reporting a decreased steal time between calls. + (patch by :user:`Arnon Yaari `) +- :gh:`1216`, [Windows]: fix compatibility with Python 2.6 (patch by + :user:`Dan Vinakovsky `) +- :gh:`1222`, [Linux]: :meth:`Process.memory_full_info` was erroneously summing + "Swap:" and "SwapPss:". Same for "Pss:" and "SwapPss". Not anymore. +- :gh:`1224`, [Windows]: :meth:`Process.wait` may erroneously raise + :exc:`TimeoutExpired`. +- :gh:`1238`, [Linux]: :func:`sensors_battery` may return ``None`` in case + battery is not listed as "BAT0" under ``/sys/class/power_supply``. +- :gh:`1240`, [Windows]: :func:`cpu_times` float loses accuracy in a long + running system. (patch by :user:`stswandering `) +- :gh:`1245`, [Linux]: :func:`sensors_temperatures` may fail with + :exc:`IOError` "no such file". +- :gh:`1255`, [FreeBSD]: :func:`swap_memory` stats were erroneously represented + in KB. (patch by :user:`Denis Krienbühl `) + +5.4.3 — 2018-01-01 +^^^^^^^^^^^^^^^^^^ + +**API changes** + +- :gh:`775`, [Windows]: :func:`disk_partitions` return mount points. + +**Bug fixes** + +- :gh:`1190`, [macOS]: :func:`pids` may return ``False`` on macOS. + +5.4.2 — 2017-12-07 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`1173`: add :envvar:`PSUTIL_DEBUG` environment variable to print debug + messages on stderr. +- :gh:`1177`, [macOS]: added support for :func:`sensors_battery`. (patch by + Arnon Yaari) + +**API changes** + +- :gh:`1188`: ``Process.memory_info_ex()`` now warns with :exc:`FutureWarning` + instead of :exc:`DeprecationWarning`. + +**Performance** + +- :gh:`1183`: :meth:`Process.children` is **2x faster** on POSIX and + **2.4x faster** on Linux. + +**Internals** + +- :gh:`1172`, [Windows]: ``make test`` does not work. + +**Bug fixes** + +- :gh:`1152`, [Windows]: :func:`disk_io_counters` may return an empty dict. +- :gh:`1169`, [Linux]: :func:`users` ``hostname`` returns username instead. + (patch by :user:`janderbrain `) +- :gh:`1179`, [Linux]: :meth:`Process.cmdline` can now split args for processes + that overwrite :proc:`/proc/pid/cmdline` with spaces instead of null bytes. +- :gh:`1181`, [macOS]: :meth:`Process.memory_maps` may raise ``ENOENT``. +- :gh:`1187`, [macOS]: :func:`pids` does not return PID 0 on recent macOS + versions. + +5.4.1 — 2017-11-08 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`1164`, [AIX]: add support for :meth:`Process.num_ctx_switches`. (patch + by Arnon Yaari) + +**Internals** + +- :gh:`1151`: ``python -m psutil.tests`` fail. + +**Dropped support** + +- :gh:`1053`, :label:`breaking`: drop Python 3.3 support (psutil still works + but it's no longer tested). + +**Bug fixes** + +- :gh:`1154`, [AIX], :label:`build-fail`: psutil won't compile on AIX 6.1.0. + (patch by Arnon Yaari) +- :gh:`1150`, [Windows]: when a process is terminated now the exit code is set + to ``SIGTERM`` instead of ``0``. (patch by :user:`Akos Kiss `) +- :gh:`1167`, [Windows]: :func:`net_io_counters` packets count now include also + non-unicast packets. (patch by :user:`Matthew Long `) + +5.4.0 — 2017-10-12 +^^^^^^^^^^^^^^^^^^ + +**New platforms** + +- :gh:`1123`, [AIX]: added support for AIX platform. (patch by + :user:`Arnon Yaari `) + +**Bug fixes** + +- :gh:`1127`, [macOS], :label:`critical`: invalid reference counting in + :meth:`Process.open_files` may lead to segfault. (patch by + :user:`Jakub Bacic `) +- :gh:`1133`, [Windows], :label:`build-fail`: can't compile on newer versions + of Visual Studio 2017 15.4. (patch by :user:`Max Bélanger `) +- :gh:`1138`, [Linux], :label:`build-fail`: can't compile on CentOS 5.0 and + RedHat 5.0. (patch by Prodesire) +- :gh:`1009`, [Linux]: :func:`sensors_temperatures` may crash with + :exc:`IOError`. +- :gh:`1012`, [Windows]: :func:`disk_io_counters` :field:`read_time` and + :field:`write_time` were expressed in tens of micro seconds instead of + milliseconds. +- :gh:`1129`, [Linux]: :func:`sensors_fans` may crash with :exc:`IOError`. + (patch by :user:`Sebastian Saip `) +- :gh:`1131`, [SunOS]: fix compilation warnings. (patch by + :user:`Arnon Yaari `) + +5.3.1 — 2017-09-10 +^^^^^^^^^^^^^^^^^^ + +**Build and packaging** + +- :gh:`1120`, [Windows], :label:`breaking`: ``.exe`` files are no longer + uploaded on PyPI as per PEP-527. Only wheels are provided. + +**Documentation** + +- :gh:`1124`: documentation moved to http://psutil.readthedocs.io + +**Bug fixes** + +- :gh:`1105`, [FreeBSD], :label:`build-fail`: psutil does not compile on + FreeBSD 12. +- :gh:`1125`, [BSD]: :func:`net_connections` raises :exc:`TypeError`. + +5.3.0 — 2017-09-01 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`802`: :func:`disk_io_counters` and :func:`net_io_counters` no longer + wrap (restart from 0). New *nowrap* argument. +- :gh:`1022`: :func:`users` provides a new :field:`pid` field. +- :gh:`1025`: :func:`process_iter` accepts new *attrs* and *ad_value* + parameters to invoke :meth:`Process.as_dict` inline. +- :gh:`1051`: :func:`disk_usage` on Python 3 is now able to accept bytes. +- :gh:`1091`, [SunOS]: implemented :meth:`Process.environ`. (patch by + :user:`Oleksii Shevchuk `) + +**API changes** + +- :gh:`1039`, :label:`breaking`: returned types consolidation. 1) Windows / + :meth:`Process.cpu_times`: fields #3 and #4 were int instead of float. 2) + Linux / FreeBSD / OpenBSD: :meth:`Process.connections` :field:`raddr` is now + set to ``""`` instead of ``None`` when retrieving UNIX sockets. +- :gh:`1040`, :label:`breaking`: all strings are encoded by using OS fs + encoding. +- :gh:`1040`, :label:`breaking`: the following Windows APIs on Python 2 now + return a string instead of unicode: :meth:`Process.memory_maps`'s + :field:`path` field, :meth:`WindowsService.binpath`, + :meth:`WindowsService.description`, :meth:`WindowsService.display_name`, + :meth:`WindowsService.username`. +- :gh:`928`: :func:`net_connections` and :meth:`Process.connections` + :field:`laddr` and :field:`raddr` are now named tuples. +- :gh:`1015`: :func:`swap_memory` now reads :proc:`/proc/meminfo` instead of + ``sysinfo()`` syscall, so it works with :data:`PROCFS_PATH` for containers. +- :gh:`1040`: implemented full unicode support. +- :gh:`1079`, [FreeBSD]: :func:`net_connections` :field:`fd` number is now + being set for real (instead of ``-1``). (patch by + :user:`Gleb Smirnoff `) + +**Build and packaging** + +- :gh:`1060`: source distribution now only includes relevant files. + +**Internals** + +- :gh:`1058`: test suite now enables all warnings by default. + +**Bug fixes** + +- :gh:`1064`, [NetBSD], :label:`critical`: :func:`swap_memory` may segfault in + case of error. +- :gh:`1042`, [FreeBSD], :label:`build-fail`: psutil won't compile on FreeBSD + 12. +- :gh:`1033`, [macOS], [FreeBSD], :label:`memleak`: memory leak for + :func:`net_connections` and :meth:`Process.connections` when retrieving UNIX + sockets (``kind='unix'``). +- :gh:`1047`, [Windows], :label:`memleak`: :meth:`Process.username`: memory + leak in case exception is thrown. +- :gh:`1050`, [Windows], :label:`memleak`: :meth:`Process.memory_maps` leaks + memory. +- :gh:`1067`, [NetBSD], :label:`memleak`: :meth:`Process.cmdline` leaks memory + if process has terminated. +- :gh:`1068`, [OpenBSD], :label:`memleak`: :func:`net_connections` leaks memory + if ``sysctl()`` fails. +- :gh:`989`, [Windows]: :func:`boot_time` may return a negative value. +- :gh:`1007`, [Windows]: :func:`boot_time` can have a 1 sec fluctuation between + calls. The first call value is now cached. +- :gh:`1013`, [FreeBSD]: :func:`net_connections` may return incorrect PID. + (patch by :user:`Gleb Smirnoff `) +- :gh:`1014`, [Linux]: :class:`Process` class can mask legitimate ``ENOENT`` + exceptions as :exc:`NoSuchProcess`. +- :gh:`1016`: :func:`disk_io_counters` raises :exc:`RuntimeError` on a system + with no disks. +- :gh:`1017`: :func:`net_io_counters` raises :exc:`RuntimeError` on a system + with no network cards installed. +- :gh:`1021`, [Linux]: :meth:`Process.open_files` may erroneously raise + :exc:`NoSuchProcess` instead of skipping a file which gets deleted while open + files are retrieved. +- :gh:`1029`, [macOS], [FreeBSD]: :meth:`Process.connections` with + ``family=unix`` on Python 3 doesn't properly handle unicode paths and may + raise :exc:`UnicodeDecodeError`. +- :gh:`1040`: fixed many unicode related issues such as + :exc:`UnicodeDecodeError` on Python 3 + POSIX and invalid encoded data on + Windows. +- :gh:`1044`, [macOS]: different :class:`Process` methods incorrectly raise + :exc:`AccessDenied` for zombie processes. +- :gh:`1046`, [Windows]: :func:`disk_partitions` on Windows overrides user's + ``SetErrorMode``. +- :gh:`1048`, [Windows]: :func:`users`'s :field:`host` field report an invalid + IP address. +- :gh:`1055`: :func:`cpu_count` is no longer cached (CPUs can be disabled at + runtime on Linux). :meth:`Process.cpu_percent` also affected. +- :gh:`1058`: fixed Python warnings. +- :gh:`1062`: :func:`disk_io_counters` and :func:`net_io_counters` raise + :exc:`TypeError` if no disks or NICs are installed on the system. +- :gh:`1063`, [NetBSD]: :func:`net_connections` may list incorrect sockets. +- :gh:`1065`, [OpenBSD]: :meth:`Process.cmdline` may raise :exc:`SystemError`. +- :gh:`1069`, [FreeBSD]: :meth:`Process.cpu_num` may return 255 for certain + kernel processes. +- :gh:`1071`, [Linux]: :func:`cpu_freq` may raise :exc:`IOError` on old RedHat + distros. +- :gh:`1074`, [FreeBSD]: :func:`sensors_battery` raises :exc:`OSError` in case + of no battery. +- :gh:`1075`, [Windows]: :func:`net_if_addrs`: ``inet_ntop()`` return value is + not checked. +- :gh:`1077`, [SunOS]: :func:`net_if_addrs` shows garbage addresses on SunOS + 5.10. (patch by :user:`Oleksii Shevchuk `) +- :gh:`1077`, [SunOS]: :func:`net_connections` does not work on SunOS 5.10. + (patch by :user:`Oleksii Shevchuk `) +- :gh:`1079`, [FreeBSD]: :func:`net_connections` didn't list locally connected + sockets. (patch by :user:`Gleb Smirnoff `) +- :gh:`1085`: :func:`cpu_count` return value is now checked and forced to + ``None`` if <= 1. +- :gh:`1087`: :meth:`Process.cpu_percent` guard against :func:`cpu_count` + returning ``None`` and assumes 1 instead. +- :gh:`1093`, [SunOS]: :meth:`Process.memory_maps` shows wrong 64 bit + addresses. +- :gh:`1094`, [Windows]: fix :func:`pid_exists` returning wrong result. All + ``OpenProcess`` APIs now verify the PID is actually running. +- :gh:`1098`, [Windows]: :meth:`Process.wait` may erroneously return sooner, + when the PID is still alive. +- :gh:`1099`, [Windows]: :meth:`Process.terminate` may raise + :exc:`AccessDenied` even if the process already died. +- :gh:`1101`, [Linux]: :func:`sensors_temperatures` may raise ``ENODEV``. + +5.2.2 — 2017-04-10 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`1000`: fixed some setup.py warnings. +- :gh:`1002`, [SunOS]: remove C macro which will not be available on new + Solaris versions. (patch by :user:`Danek Duvall `) +- :gh:`1004`, [Linux]: :meth:`Process.io_counters` may raise :exc:`ValueError`. +- :gh:`1006`, [Linux]: :func:`cpu_freq` may return ``None`` on some Linux + versions does not support the function. Let's not make the function available + instead. +- :gh:`1009`, [Linux]: :func:`sensors_temperatures` may raise :exc:`OSError`. +- :gh:`1010`, [Linux]: :func:`virtual_memory` may raise :exc:`ValueError` on + Ubuntu 14.04. + +5.2.1 — 2017-03-24 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`981`, [Linux]: :func:`cpu_freq` may return an empty list. +- :gh:`993`, [Windows]: :meth:`Process.memory_maps` on Python 3 may raise + :exc:`UnicodeDecodeError`. +- :gh:`996`, [Linux]: :func:`sensors_temperatures` may not show all + temperatures. +- :gh:`997`, [FreeBSD]: :func:`virtual_memory` may fail due to missing + ``sysctl`` parameter on FreeBSD 12. + +5.2.0 — 2017-03-05 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`974`, [Linux]: Add :func:`sensors_fans` function. (patch by + :user:`Nicolas Hennion `) +- :gh:`976`, [Windows]: :meth:`Process.io_counters` has 2 new fields: + :field:`other_count` and :field:`other_bytes`. +- :gh:`976`, [Linux]: :meth:`Process.io_counters` has 2 new fields: + :field:`read_chars` and :field:`write_chars`. + +**Bug fixes** + +- :gh:`985`, [Windows], :label:`critical`: Fix a crash in + :meth:`Process.open_files` when the worker thread for ``NtQueryObject`` times + out. +- :gh:`872`, [Linux], :label:`build-fail`: can now compile on Linux by using + MUSL C library. +- :gh:`986`, [Linux]: :meth:`Process.cwd` may raise :exc:`NoSuchProcess` + instead of :exc:`ZombieProcess`. + +5.1.3 — 2017-02-07 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`971`, [Linux]: :func:`sensors_temperatures` didn't work on CentOS 7. +- :gh:`973`: :func:`cpu_percent` may raise :exc:`ZeroDivisionError`. + +5.1.2 — 2017-02-03 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`966`, [Linux]: :func:`sensors_battery` :field:`power_plugged` may + erroneously return ``None`` on Python 3. +- :gh:`968`, [Linux]: :func:`disk_io_counters` raises :exc:`TypeError` on + Python 3. +- :gh:`970`, [Linux]: :func:`sensors_battery` :field:`name` and :field:`label` + fields on Python 3 are bytes instead of str. + +5.1.1 — 2017-02-03 +^^^^^^^^^^^^^^^^^^ + +**API changes** + +- :gh:`966`, [Linux]: :func:`sensors_battery` :field:`percent` is a float and + is more precise. + +**Bug fixes** + +- :gh:`964`, [Windows]: :meth:`Process.username` and :func:`users` may return + badly decoded character on Python 3. +- :gh:`965`, [Linux]: :func:`disk_io_counters` may miscalculate sector size and + report the wrong number of bytes read and written. +- :gh:`966`, [Linux]: :func:`sensors_battery` may fail with + :exc:`FileNotFoundError`. +- :gh:`966`, [Linux]: :func:`sensors_battery` :field:`power_plugged` may lie. + +5.1.0 — 2017-02-01 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`357`: added :meth:`Process.cpu_num` (what CPU a process is on). +- :gh:`371`, [Linux]: added :func:`sensors_temperatures`. +- :gh:`941`: added :func:`cpu_freq` (CPU frequency). +- :gh:`955`, [Linux], [Windows]: added :func:`sensors_battery`. +- :gh:`956`: :meth:`Process.cpu_affinity` can now be passed ``[]`` argument as + an alias to set affinity against all eligible CPUs. + +**Bug fixes** + +- :gh:`948`, :label:`build-fail`: cannot install psutil with + ``PYTHONOPTIMIZE=2``. +- :gh:`687`, [Linux]: :func:`pid_exists` no longer returns ``True`` if passed a + process thread ID. +- :gh:`950`, [Windows]: :meth:`Process.cpu_percent` was calculated incorrectly + and showed higher number than real usage. +- :gh:`951`, [Windows]: the uploaded wheels for Python 3.6 64 bit didn't work. +- :gh:`959`: psutil exception objects could not be pickled. +- :gh:`960`: :class:`Popen` ``wait()`` did not return the correct negative exit + status if process is killed by a signal. +- :gh:`961`, [Windows]: :meth:`WindowsService.description` method may fail with + ``ERROR_MUI_FILE_NOT_FOUND``. + +5.0.1 — 2016-12-21 +^^^^^^^^^^^^^^^^^^ + +**Build and packaging** + +- :gh:`939`: tar.gz distribution went from 1.8M to 258K. + +**Bug fixes** + +- :gh:`609`, [SunOS], :label:`build-fail`: psutil does not compile on Solaris + 10. +- :gh:`936`, [Windows], :label:`build-fail`: fix compilation error on VS 2013 + (patch by :user:`Max Bélanger `). +- :gh:`811`, [Windows]: provide a more meaningful error message if trying to + use psutil on unsupported Windows XP. +- :gh:`940`, [Linux]: :func:`cpu_percent` and :func:`cpu_times_percent` was + calculated incorrectly as :field:`iowait`, :field:`guest` and + :field:`guest_nice` times were not properly taken into account. +- :gh:`944`, [OpenBSD]: :func:`pids` was omitting PID 0. + +5.0.0 — 2016-11-06 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`799`: new :meth:`Process.oneshot` context manager (**+2x faster** in + general, **+2x to +6x** on Windows). + +**Bug fixes** + +- :gh:`933`, [Windows], :label:`memleak`: memory leak in :func:`cpu_stats` and + :meth:`WindowsService.description` method. +- :gh:`932`, [NetBSD]: :func:`net_connections` and :meth:`Process.connections` + may fail without raising an exception. +- :gh:`943`: better error message in case of version conflict on import. + +4.4.2 — 2016-10-26 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`931`, [SunOS], :label:`build-fail`: psutil no longer compiles on + Solaris. + +4.4.1 — 2016-10-25 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`927`: :class:`Popen` ``__del__`` may cause maximum recursion depth + error. + +4.4.0 — 2016-10-23 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`874`, [Windows]: make :func:`net_if_addrs` also return the + :field:`netmask`. + +**API changes** + +- :gh:`887`, [Linux]: :func:`virtual_memory` :field:`available` and + :field:`used` are more precise and match ``free`` utility. Also handles LXC + containers. + +**Internals** + +- :gh:`891`: :src:`scripts/procinfo.py` has been updated and provides a lot + more info. + +**Bug fixes** + +- :gh:`514`, [macOS], :label:`critical`: :meth:`Process.memory_maps` can + segfault. +- :gh:`926`, [macOS], :label:`critical`: :meth:`Process.environ` can crash the + interpreter on Python 3 if the process environment contains an invalid + unicode string. +- :gh:`783`, [macOS]: :meth:`Process.status` may erroneously return + :data:`STATUS_RUNNING` for zombie processes. +- :gh:`798`, [Windows]: :meth:`Process.open_files` returns and empty list on + Windows 10. +- :gh:`825`, [Linux]: :meth:`Process.cpu_affinity`: fix possible double close + and use of unopened socket. +- :gh:`880`, [Windows]: fix race condition inside :func:`net_connections`. +- :gh:`885`: :exc:`ValueError` is raised if a negative integer is passed to + :func:`cpu_percent` functions. +- :gh:`892`, [Linux]: :meth:`Process.cpu_affinity` with ``[-1]`` as arg raises + :exc:`SystemError` with no error set; now :exc:`ValueError` is raised. +- :gh:`906`, [BSD]: :func:`disk_partitions` with ``all=False`` returned an + empty list. Now the argument is ignored and all partitions are always + returned. +- :gh:`907`, [FreeBSD]: :meth:`Process.exe` may fail with :exc:`OSError` + ``ENOENT``. +- :gh:`908`, [macOS], [BSD]: different process methods could errounesuly mask + the real error for high-privileged PIDs and raise :exc:`NoSuchProcess` and + :exc:`AccessDenied` instead of :exc:`OSError` and :exc:`RuntimeError`. +- :gh:`909`, [macOS]: :meth:`Process.open_files` and + :meth:`Process.connections` methods may raise :exc:`OSError` with no + exception set if process is gone. +- :gh:`916`, [macOS]: fix many compilation warnings. + +4.3.1 — 2016-09-01 +^^^^^^^^^^^^^^^^^^ + +**Build and packaging** + +- :gh:`881`: ``make install`` now works also when using a virtual env. + +**Bug fixes** + +- :gh:`870`, [Windows], :label:`memleak`: handle leak inside + ``psutil_get_process_data``. +- :gh:`854`: :meth:`Process.as_dict` raises :exc:`ValueError` if passed an + erroneous attrs name. +- :gh:`857`, [SunOS]: :meth:`Process.cpu_times`, :meth:`Process.cpu_percent`, + :meth:`Process.threads` and :meth:`Process.memory_maps` may raise + :exc:`RuntimeError` if attempting to query a 64bit process with a 32bit + Python. "Null" values are returned as a fallback. +- :gh:`858`: :meth:`Process.as_dict` should not call + ``Process.memory_info_ex()`` because it's deprecated. +- :gh:`863`, [Windows]: :meth:`Process.memory_maps` truncates addresses above + 32 bits. +- :gh:`866`, [Windows]: :func:`win_service_iter` and services in general are + not able to handle unicode service names / descriptions. +- :gh:`869`, [Windows]: :meth:`Process.wait` may raise :exc:`TimeoutExpired` + with wrong timeout unit (ms instead of sec). + +4.3.0 — 2016-06-18 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`812`, [NetBSD], :label:`build-fail`: fix compilation on NetBSD-5.x. +- :gh:`810`, [Windows]: Windows wheels are incompatible with pip 7.1.2. +- :gh:`823`, [NetBSD]: :func:`virtual_memory` raises :exc:`TypeError` on Python + 3. +- :gh:`829`, [POSIX]: :func:`disk_usage` :field:`percent` field takes root + reserved space into account. +- :gh:`816`, [Windows]: fixed :func:`net_io_counters` values wrapping after + 4.3GB in Windows Vista (NT 6.0) and above using 64bit values from newer win + APIs. + +4.2.0 — 2016-05-14 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`795`, [Windows]: new APIs to deal with Windows services: + :func:`win_service_iter` and :func:`win_service_get`. +- :gh:`800`, [Linux]: :func:`virtual_memory` returns a new :field:`shared` + field. + +**Performance** + +- :gh:`819`, [Linux]: speedup ``/proc`` parsing: :meth:`Process.ppid` +20% + faster. :meth:`Process.status` **+28% faster**. :meth:`Process.name` + **+25% faster**. :meth:`Process.num_threads` **+20% faster** on Python 3. + +**Bug fixes** + +- :gh:`797`, [Linux]: :func:`net_if_stats` may raise :exc:`OSError` for certain + NIC cards. +- :gh:`813`: :meth:`Process.as_dict` should ignore extraneous attribute names + which gets attached to the :class:`Process` instance. + +4.1.0 — 2016-03-12 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`777`, [Linux]: :meth:`Process.open_files` on Linux return 3 new fields: + :field:`position`, :field:`mode` and :field:`flags`. +- :gh:`779`: :meth:`Process.cpu_times` returns two new fields, + :field:`children_user` and :field:`children_system` (always set to 0 on macOS + and Windows). +- :gh:`789`, [Windows]: :func:`cpu_times` return two new fields: + :field:`interrupt` and :field:`dpc`. Same for :func:`cpu_times_percent`. +- :gh:`792`: new :func:`cpu_stats` function returning number of CPU + :field:`ctx_switches`, :field:`interrupts`, :field:`soft_interrupts` and + :field:`syscalls`. + +**Bug fixes** + +- :gh:`780`, [macOS], :label:`build-fail`: psutil does not compile with some + GCC versions. +- :gh:`790`, [macOS], :label:`build-fail`: psutil won't compile on macOS 10.4. +- :gh:`774`, [FreeBSD]: :func:`net_io_counters` dropout is no longer set to 0 + if the kernel provides it. +- :gh:`776`, [Linux]: :meth:`Process.cpu_affinity` may erroneously raise + :exc:`NoSuchProcess`. (patch by :user:`wxwright`) +- :gh:`786`: :func:`net_if_addrs` may report incomplete MAC addresses. +- :gh:`788`, [NetBSD]: :func:`virtual_memory` :field:`buffers` and + :field:`shared` values were set to 0. + +4.0.0 — 2016-02-17 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`523`, [Linux], [FreeBSD]: :func:`disk_io_counters` return a new + :field:`busy_time` field. +- :gh:`732`: :meth:`Process.environ`. (patch by + :user:`Frank Benkstein `) +- :gh:`753`, [Linux], [macOS], [Windows]: process :term:`USS` and :term:`PSS` + (Linux) "real" memory stats. (patch by :user:`Eric Rahm `) +- :gh:`755`: :meth:`Process.memory_percent` ``memtype`` parameter. +- :gh:`760`: expose OS constants (:data:`LINUX`, :data:`OSX`, etc.) +- :gh:`756`, [Linux]: :func:`disk_io_counters` return 2 new fields: + :field:`read_merged_count` and :field:`write_merged_count`. + +**Build and packaging** + +- :gh:`660`, [Windows]: make.bat is smarter in finding alternative VS install + locations. (patch by :user:`mpderbec`) + +**Internals** + +- :gh:`758`: tests now live in psutil namespace. +- :gh:`762`: add :src:`scripts/procsmem.py`. + +**Bug fixes** + +- :gh:`751`, [Linux], :label:`critical`: fixed call to ``Py_DECREF`` on + possible ``NULL`` object. +- :gh:`704`, [SunOS], :label:`build-fail`: psutil does not compile on Solaris + sparc. +- :gh:`741`, [OpenBSD], :label:`build-fail`: psutil does not compile on mips64. +- :gh:`764`, [NetBSD], :label:`build-fail`: fix compilation on NetBSD-6.x. +- :gh:`685`, [Linux]: :func:`virtual_memory` provides wrong results on systems + with a lot of physical memory. +- :gh:`734`: on Python 3 invalid UTF-8 data is not correctly handled for + :meth:`Process.name`, :meth:`Process.cwd`, :meth:`Process.exe`, + :meth:`Process.cmdline` and :meth:`Process.open_files` methods resulting in + :exc:`UnicodeDecodeError` exceptions. ``'surrogateescape'`` error handler is + now used as a workaround for replacing the corrupted data. +- :gh:`737`, [Windows]: when the bitness of psutil and the target process was + different, :meth:`Process.cmdline` and :meth:`Process.cwd` could return a + wrong result or incorrectly report an :exc:`AccessDenied` error. +- :gh:`754`, [Linux]: :meth:`Process.cmdline` can be wrong in case of zombie + process. +- :gh:`759`, [Linux]: :meth:`Process.memory_maps` may return paths ending with + ``" (deleted)"``. +- :gh:`761`, [Windows]: :func:`boot_time` wraps to 0 after 49 days. +- :gh:`766`, [Linux]: :func:`net_connections` can't handle malformed + ``/proc/net/unix`` file. +- :gh:`767`, [Linux]: :func:`disk_io_counters` may raise :exc:`ValueError` on + 2.6 kernels and it's broken on 2.4 kernels. +- :gh:`770`, [NetBSD]: :func:`disk_io_counters` metrics didn't update. + +3.4.2 — 2016-01-20 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`728`, [SunOS]: exposed :data:`PROCFS_PATH` constant to change the + default location of ``/proc`` filesystem. + +**Bug fixes** + +- :gh:`724`, [FreeBSD]: :func:`virtual_memory` :field:`total` is incorrect. +- :gh:`730`, [FreeBSD]: :func:`virtual_memory` crashes with "OSError: [Errno + 12] Cannot allocate memory". + +3.4.1 — 2016-01-15 +^^^^^^^^^^^^^^^^^^ + +**New platforms** + +- :gh:`557`, [NetBSD]: added NetBSD support. (contributed by Ryo Onodera and + Thomas Klausner) + +**API changes** + +- :gh:`718`: :func:`process_iter` is now thread safe. + +**Performance** + +- :gh:`708`, [Linux]: :func:`net_connections` and :meth:`Process.connections` + on Python 2 can be up to **3x faster** in case of many connections. Also + :meth:`Process.memory_maps` is slightly faster. + +**Bug fixes** + +- :gh:`715`, :label:`critical`: don't crash at import time if :func:`cpu_times` + fail for some reason. +- :gh:`714`, [OpenBSD]: :func:`virtual_memory` :field:`cached` value was always + set to 0. +- :gh:`717`, [Linux]: :meth:`Process.open_files` fails if deleted files still + visible. +- :gh:`722`, [Linux]: :func:`swap_memory` no longer crashes if :field:`sin` / + :field:`sout` can't be determined due to missing :proc:`/proc/vmstat`. +- :gh:`724`, [FreeBSD]: :func:`virtual_memory` :field:`total` is slightly + incorrect. + +3.3.0 — 2015-11-25 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`558`, [Linux]: exposed :data:`PROCFS_PATH` constant to change the + default location of ``/proc`` filesystem. + +**New platforms** + +- :gh:`615`, [OpenBSD]: added OpenBSD support. (contributed by Landry Breuil) + +**Bug fixes** + +- :gh:`692`, [POSIX]: :meth:`Process.name` is no longer cached as it may + change. + +3.2.2 — 2015-10-04 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`610`, [SunOS], :label:`build-fail`: fix build and tests on Solaris 10 +- :gh:`678`, [Linux], :label:`build-fail`: can't install psutil due to bug in + setup.py. +- :gh:`688`, [Windows], :label:`build-fail`: compilation fails with MSVC 2015, + Python 3.5. (patch by Mike Sarahan) +- :gh:`690`, [Windows], :label:`build-fail`: can't compile with MSVC 2015, + which enforces the C++11 rule forbidding duplicate values in an unscoped + enum. +- :gh:`517`, [SunOS]: :func:`net_io_counters` failed to detect network + interfaces correctly on Solaris 10 +- :gh:`541`, [FreeBSD]: :func:`disk_io_counters` r/w times were expressed in + seconds instead of milliseconds. (patch by :user:`dasumin `) +- :gh:`623`, [Linux]: process or system connections raises :exc:`ValueError` if + IPv6 is not supported by the system. + +3.2.1 — 2015-09-03 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`677`, [Linux], :label:`build-fail`: can't install psutil due to bug in + setup.py. + +3.2.0 — 2015-09-02 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`644`, [Windows]: added support for ``CTRL_C_EVENT`` and + ``CTRL_BREAK_EVENT`` signals to use with :meth:`Process.send_signal`. + +**API changes** + +- :gh:`663`, [POSIX]: :func:`net_if_addrs` now returns point-to-point (VPNs) + addresses. +- :gh:`655`, [Windows]: fix various unicode handling issues. On Python 2, + string APIs now return encoded strings using + :func:`sys.getfilesystemencoding`. + +**Internals** + +- :gh:`648`, [macOS]: CI test integration. (patch by + :user:`Jeff Tang `) + +**Bug fixes** + +- :gh:`670`, [Windows], :label:`critical`: segfault of :func:`net_if_addrs` in + case of non-ASCII NIC names. (patch by :user:`sk6249 `) +- :gh:`659`, [Linux], :label:`build-fail`: compilation error on Suse 10. (patch + by :user:`maozguttman`) +- :gh:`664`, [Linux], :label:`build-fail`: compilation error on Alpine Linux. + (patch by :user:`Bart van Kleef `) +- :gh:`672`, [Windows], :label:`build-fail`: compilation fails if using Windows + SDK v8.0. (patch by Steven Winfield) +- :gh:`513`, [Linux]: fixed integer overflow for :data:`RLIM_INFINITY` +- :gh:`641`, [Windows]: fixed many compilation warnings. (patch by + :user:`Jeff Tang `) +- :gh:`652`, [Windows]: :func:`net_if_addrs` :exc:`UnicodeDecodeError` in case + of non-ASCII NIC names. +- :gh:`655`, [Windows]: :func:`net_if_stats` :exc:`UnicodeDecodeError` in case + of non-ASCII NIC names. +- :gh:`675`, [Linux]: :func:`net_connections`: :exc:`UnicodeDecodeError` may + occur when listing UNIX sockets. + +3.1.1 — 2015-07-15 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`603`, [Linux]: :meth:`Process.ionice` set value range is incorrect. + (patch by :user:`spacewander `) +- :gh:`645`, [Linux]: :func:`cpu_times_percent` may produce negative results. +- :gh:`656`: ``from psutil import *`` does not work. + +3.1.0 — 2015-07-15 +^^^^^^^^^^^^^^^^^^ + +**API changes** + +- :gh:`534`, [Linux]: :func:`disk_partitions` added support for ZFS + filesystems. + +**Documentation** + +- :gh:`647`: new dev guide: + https://github.com/giampaolo/psutil/blob/master/docs/devguide.rst + +**Internals** + +- :gh:`646`, [Windows]: continuous tests integration for Windows with + https://ci.appveyor.com/project/giampaolo/psutil. +- :gh:`651`: continuous code quality test integration with scrutinizer-ci.com + +**Bug fixes** + +- :gh:`340`, [Windows], :label:`critical`: :meth:`Process.open_files` no longer + hangs (uses a thread with timeout). (patch by :user:`Jeff Tang `) +- :gh:`627`, [Windows]: :meth:`Process.name` no longer raises + :exc:`AccessDenied` for pids owned by another user. +- :gh:`636`, [Windows]: :meth:`Process.memory_info` raise :exc:`AccessDenied`. +- :commit:`5ae30c79`, [POSIX]: raise exception if trying to send signal to PID + 0 as it will affect :func:`os.getpid` 's process group and not PID 0. +- :gh:`639`, [Linux]: :meth:`Process.cmdline` can be truncated. +- :gh:`640`, [Linux]: ``*connections`` functions may swallow errors and return + an incomplete list of connections. +- :gh:`642`: ``repr()`` of exceptions is incorrect. +- :gh:`653`, [Windows]: add ``inet_ntop()`` function for Windows XP to support + IPv6. +- :gh:`641`, [Windows]: replace deprecated string functions with safe + equivalents. + +3.0.1 — 2015-06-18 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`635`, [POSIX], :label:`critical`: crash on module import if :mod:`enum` + package is installed on Python < 3.4. +- :gh:`632`, [Linux]: better error message if cannot parse process UNIX + connections. +- :gh:`634`, [Linux]: :meth:`Process.cmdline` does not include empty string + arguments. + +3.0.0 — 2015-06-13 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`250`: new :func:`net_if_stats` returning NIC statistics (:field:`isup`, + :field:`duplex`, :field:`speed`, :field:`mtu`). +- :gh:`376`: new :func:`net_if_addrs` returning all NIC addresses a-la + ``ifconfig``. + +**API changes** + +- :gh:`594`, :label:`breaking`: all deprecated APIs were removed. +- :gh:`469`: on Python >= 3.4 ``IOPRIO_CLASS_*`` and ``*_PRIORITY_CLASS`` + constants returned by :meth:`Process.ionice` and :meth:`Process.nice` are + enums instead of plain integers. +- :gh:`582`: connection constants returned by :func:`net_connections` and + :meth:`Process.connections` were turned from int to enums on Python > 3.4. +- :gh:`589`: :meth:`Process.cpu_affinity` accepts any kind of iterable (set, + tuple, ...), not only lists. +- :gh:`599`, [Windows]: :meth:`Process.name` can now be determined for all + processes even when running as a limited user. + +**Build and packaging** + +- :gh:`587`: move native extension into the package. + +**Internals** + +- :gh:`581`: add ``.gitignore``. (patch by :user:`Gabi Davar `) +- :gh:`602`: pre-commit GIT hook. +- :gh:`629`: enhanced support for ``pytest`` and ``nose`` test runners. +- :gh:`616`, [Windows]: add ``inet_ntop()`` function for Windows XP. + +**Bug fixes** + +- :gh:`512`, [BSD], :label:`critical`: fix segfault in :func:`net_connections`. +- :gh:`586`, [FreeBSD], :label:`critical`: :meth:`Process.cpu_affinity` + segfaults on set in case an invalid CPU number is provided. +- :gh:`593`, [FreeBSD], :label:`critical`: :meth:`Process.memory_maps` + segfaults. +- :gh:`607`, [Linux], :label:`build-fail`: can't compile on old RedHat versions + where ``DUPLEX_UNKNOWN`` is not defined. +- :gh:`428`, [POSIX]: correct handling of zombie processes on POSIX. Introduced + new :exc:`ZombieProcess` exception class. +- :gh:`555`, [Linux]: :func:`users` correctly handles ``":0"`` as an alias for + ``"localhost"``. +- :gh:`579`, [Windows]: fixed :meth:`Process.open_files` for PID > 64K. +- :gh:`579`, [Windows]: fixed many compiler warnings. +- :gh:`585`, [FreeBSD]: :func:`net_connections` may raise :exc:`KeyError`. +- :gh:`606`: :meth:`Process.parent` may swallow :exc:`NoSuchProcess` + exceptions. +- :gh:`611`, [SunOS]: :func:`net_io_counters` has send and received swapped +- :gh:`614`, [Linux]:: :func:`cpu_count` with ``logical=False`` return the + number of sockets instead of cores. +- :gh:`618`, [SunOS]: swap tests fail on Solaris when run as normal user. +- :gh:`628`, [Linux]: :meth:`Process.name` truncates string in case it contains + spaces or parentheses. + +2.2.1 — 2015-02-02 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`572`, [Linux]: fix "ValueError: ambiguous inode with multiple PIDs + references" for :meth:`Process.connections`. (patch by + :user:`Bruno Binet `) + +2.2.0 — 2015-01-06 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`569`, [FreeBSD]: add support for :meth:`Process.cpu_affinity` on + FreeBSD. + +**Internals** + +- :gh:`553`: add :src:`scripts/pstree.py`. +- :gh:`568`: add :src:`scripts/pidof.py`. + +**Dropped support** + +- :gh:`521`, :label:`breaking`: drop support for Python 2.4 and 2.5. + +**Bug fixes** + +- :gh:`496`, [SunOS], :label:`critical`: can't import psutil. +- :gh:`556`, [Linux], :label:`memleak`: lots of file handles were left open. +- :gh:`569`, [FreeBSD], :label:`memleak`: fix memory leak in :func:`cpu_count` + with ``logical=False``. +- :gh:`547`, [POSIX]: :meth:`Process.username` may raise :exc:`KeyError` if UID + can't be resolved. +- :gh:`551`, [Windows]: get rid of the unicode hack for :func:`net_io_counters` + NIC names. +- :gh:`561`, [Linux]: :func:`net_connections` might skip some legitimate UNIX + sockets. (patch by :user:`spacewander `) +- :gh:`564`: C extension version mismatch is now detected at import time. +- :gh:`565`, [Windows]: use proper encoding for :meth:`Process.username` and + :func:`users`. (patch by :user:`Sylvain Mouquet `) +- :gh:`567`, [Linux]: in the alternative implementation of + :meth:`Process.cpu_affinity` ``PyList_Append`` and ``Py_BuildValue`` return + values are not checked. +- :gh:`571`, [Linux]: :meth:`Process.open_files` might swallow + :exc:`AccessDenied` exceptions and return an incomplete list of open files. + +2.1.3 — 2014-09-26 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`536`, [Linux], :label:`build-fail`: fix "undefined symbol: CPU_ALLOC" + compilation error. + +2.1.2 — 2014-09-21 +^^^^^^^^^^^^^^^^^^ + +**Build and packaging** + +- :gh:`505`, [Windows]: distribution as wheel packages. + +**Internals** + +- :gh:`407`: project moved from Google Code to Github; code moved from + Mercurial to Git. +- :gh:`492`: use ``tox`` to run tests on multiple Python versions. (patch by + msabramo) +- :gh:`511`: add :src:`scripts/ps.py`. + +**Bug fixes** + +- :gh:`340`, [Windows], :label:`critical`: :meth:`Process.open_files` no longer + hangs. (patch by Jeff Tang) +- :gh:`501`, [Windows]: :func:`disk_io_counters` may return negative values. +- :gh:`503`, [Linux]: in rare conditions :meth:`Process.exe`, + :meth:`Process.open_files` and :meth:`Process.connections` can raise + ``OSError(ESRCH)`` instead of :exc:`NoSuchProcess`. +- :gh:`504`, [Linux]: can't build RPM packages via setup.py +- :gh:`506`, [Linux]: Python 2.4 support was broken. +- :gh:`522`, [Linux]: :meth:`Process.cpu_affinity` might return ``EINVAL``. + (patch by :user:`David Daeschler `) +- :gh:`529`, [Windows]: :meth:`Process.exe` may raise unhandled + :exc:`WindowsError` exception for PIDs 0 and 4. (patch by + :user:`Jeff Tang `) +- :gh:`530`, [Linux]: :func:`disk_io_counters` may crash on old Linux distros + (< 2.6.5) (patch by :user:`Yaolong Huang `) +- :gh:`533`, [Linux]: :meth:`Process.memory_maps` may raise :exc:`TypeError` on + old Linux distros. + +2.1.1 — 2014-04-30 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`446`, [Windows]: fix encoding error when using :func:`net_io_counters` + on Python 3. (patch by :user:`Szigeti Gabor Niif `) +- :gh:`460`, [Windows]: :func:`net_io_counters` wraps after 4G. +- :gh:`497`, [Linux]: :func:`net_connections` exceptions. (patch by + :user:`Alexander Grothe `) + +2.1.0 — 2014-04-08 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`387`: system-wide open connections a-la ``netstat`` (add + :func:`net_connections`). + +**Bug fixes** + +- :gh:`421`, [SunOS], :label:`build-fail`: psutil does not compile on SunOS + 5.10. (patch by Naveed Roudsari) +- :gh:`489`, [Linux]: :func:`disk_partitions` return an empty list. + +2.0.0 — 2014-03-10 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`427`: add :func:`cpu_count`. + +**API changes** + +- :gh:`447`: :func:`wait_procs` *timeout* parameter is now optional. +- :gh:`452`: make :class:`Process` instances hashable and usable with ``set()`` + s. + +For the sake of consistency a lot of psutil APIs have been renamed. In most +cases accessing the old names will work but it will cause a +:exc:`DeprecationWarning`. + +- ``psutil.*`` module level constants have being replaced by functions: + + +-----------------------+----------------------------------+ + | Old name | Replacement | + +=======================+==================================+ + | psutil.NUM_CPUS | psutil.cpu_count() | + +-----------------------+----------------------------------+ + | psutil.BOOT_TIME | psutil.boot_time() | + +-----------------------+----------------------------------+ + | psutil.TOTAL_PHYMEM | virtual_memory.total | + +-----------------------+----------------------------------+ + +- Renamed ``psutil.*`` functions: + + +------------------------+-------------------------------+ + | Old name | Replacement | + +========================+===============================+ + | psutil.get_pid_list() | psutil.pids() | + +------------------------+-------------------------------+ + | psutil.get_users() | psutil.users() | + +------------------------+-------------------------------+ + | psutil.get_boot_time() | psutil.boot_time() | + +------------------------+-------------------------------+ + +- All :class:`Process` ``get_*`` methods lost the ``get_`` prefix. E.g. + ``get_ext_memory_info()`` was renamed to ``memory_info_ex()``. Assuming + ``p = psutil.Process()``: + + +--------------------------+----------------------+ + | Old name | Replacement | + +==========================+======================+ + | p.get_children() | p.children() | + +--------------------------+----------------------+ + | p.get_connections() | p.connections() | + +--------------------------+----------------------+ + | p.get_cpu_affinity() | p.cpu_affinity() | + +--------------------------+----------------------+ + | p.get_cpu_percent() | p.cpu_percent() | + +--------------------------+----------------------+ + | p.get_cpu_times() | p.cpu_times() | + +--------------------------+----------------------+ + | p.get_ext_memory_info() | p.memory_info_ex() | + +--------------------------+----------------------+ + | p.get_io_counters() | p.io_counters() | + +--------------------------+----------------------+ + | p.get_ionice() | p.ionice() | + +--------------------------+----------------------+ + | p.get_memory_info() | p.memory_info() | + +--------------------------+----------------------+ + | p.get_memory_maps() | p.memory_maps() | + +--------------------------+----------------------+ + | p.get_memory_percent() | p.memory_percent() | + +--------------------------+----------------------+ + | p.get_nice() | p.nice() | + +--------------------------+----------------------+ + | p.get_num_ctx_switches() | p.num_ctx_switches() | + +--------------------------+----------------------+ + | p.get_num_fds() | p.num_fds() | + +--------------------------+----------------------+ + | p.get_num_threads() | p.num_threads() | + +--------------------------+----------------------+ + | p.get_open_files() | p.open_files() | + +--------------------------+----------------------+ + | p.get_rlimit() | p.rlimit() | + +--------------------------+----------------------+ + | p.get_threads() | p.threads() | + +--------------------------+----------------------+ + | p.getcwd() | p.cwd() | + +--------------------------+----------------------+ + +- All :class:`Process` ``set_*`` methods lost the ``set_`` prefix. Assuming + ``p = psutil.Process()``: + + +----------------------+---------------------------------+ + | Old name | Replacement | + +======================+=================================+ + | p.set_nice() | p.nice(value) | + +----------------------+---------------------------------+ + | p.set_ionice() | p.ionice(ioclass, value=None) | + +----------------------+---------------------------------+ + | p.set_cpu_affinity() | p.cpu_affinity(cpus) | + +----------------------+---------------------------------+ + | p.set_rlimit() | p.rlimit(resource, limits=None) | + +----------------------+---------------------------------+ + +- Except for ``pid``, all :class:`Process` class properties have been turned + into methods. This is the only case which there are no aliases. Assuming + ``p = psutil.Process()``: + + +---------------+-----------------+ + | Old name | Replacement | + +===============+=================+ + | p.name | p.name() | + +---------------+-----------------+ + | p.parent | p.parent() | + +---------------+-----------------+ + | p.ppid | p.ppid() | + +---------------+-----------------+ + | p.exe | p.exe() | + +---------------+-----------------+ + | p.cmdline | p.cmdline() | + +---------------+-----------------+ + | p.status | p.status() | + +---------------+-----------------+ + | p.uids | p.uids() | + +---------------+-----------------+ + | p.gids | p.gids() | + +---------------+-----------------+ + | p.username | p.username() | + +---------------+-----------------+ + | p.create_time | p.create_time() | + +---------------+-----------------+ + +- :gh:`479`, :label:`breaking`: long deprecated ``psutil.error`` module is + gone; exception classes now live in "psutil" namespace only. +- :gh:`463`: *timeout* parameter of ``cpu_percent*`` functions defaults to 0.0 + instead of 0.1, avoiding a common source of accidental slowdowns. +- :class:`Process` instances' ``retcode`` attribute returned by + :func:`wait_procs` has been renamed to ``returncode`` for consistency with + :class:`subprocess.Popen`. + +**Performance** + +- :gh:`477`: :meth:`Process.cpu_percent` is about **30% faster**. (suggested by + crusaderky) +- :gh:`478`, [Linux]: almost all APIs are about **30% faster** on Python 3.X. + +**Build and packaging** + +- :gh:`424`, [Windows]: installer for Python 3.X 64 bit. + +**Documentation** + +- :gh:`468`: move documentation to readthedocs.com. + +**Internals** + +- :gh:`453`: tests on Python < 2.7 require ``unittest2`` module. +- :gh:`459`: add a Makefile for running tests and other repetitive tasks (also + on Windows). + +**Bug fixes** + +- :gh:`340`, [Windows], :label:`critical`: :meth:`Process.open_files` no longer + hangs. (patch by jtang@vahna.net) +- :gh:`448`, [Windows], :label:`memleak`: :meth:`Process.children` and + :meth:`Process.ppid` memory leak (patch by Ulrich Klank). +- :gh:`193`: :class:`Popen` constructor can throw an exception if the spawned + process terminates quickly. +- :gh:`443`, [Linux]: fix a potential overflow issue for + :meth:`Process.cpu_affinity` (set) on systems with more than 64 CPUs. +- :gh:`457`, [POSIX]: :func:`pid_exists` always returns ``True`` for PID 0. +- :gh:`461`: named tuples are not pickle-able. +- :gh:`466`, [Linux]: :meth:`Process.exe` improper null bytes handling. (patch + by Gautam Singh) +- :gh:`470`: :func:`wait_procs` might not wait. (patch by :user:`crusaderky`) +- :gh:`471`, [Windows]: :meth:`Process.exe` improper unicode handling. (patch + by alex@mroja.net) +- :gh:`473`: :class:`Popen` ``wait()`` method does not set ``returncode`` + attribute. +- :gh:`474`, [Windows]: :meth:`Process.cpu_percent` is no longer capped at + 100%. +- :gh:`476`, [Linux]: encoding error for :meth:`Process.name` and + :meth:`Process.cmdline`. + +1.2.1 — 2013-11-25 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`348`, [Windows], :label:`critical`: fixed "ImportError: DLL load failed" + occurring on module import on Windows XP. +- :gh:`425`, [SunOS], :label:`critical`: crash on import due to failure at + determining ``BOOT_TIME``. +- :gh:`443`, [Linux]: :meth:`Process.cpu_affinity` can't set affinity on + systems with more than 64 cores. + +1.2.0 — 2013-11-20 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`440`: new :func:`wait_procs` utility function which waits for multiple + processes to terminate. + +**API changes** + +- :gh:`439`: assume :func:`os.getpid` if no argument is passed to + :class:`Process` class constructor. + +**Bug fixes** + +- :gh:`348`, [Windows], :label:`critical`: fix "ImportError: DLL load failed" + occurring on module import on Windows XP / Vista. + +1.1.3 — 2013-11-07 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`442`, [Linux], :label:`build-fail`: psutil won't compile on certain + version of Linux because of missing :manpage:`prlimit(2)` syscall. + +1.1.2 — 2013-10-22 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`442`, [Linux], :label:`build-fail`: psutil won't compile on Debian 6.0 + because of missing :manpage:`prlimit(2)` syscall. + +1.1.1 — 2013-10-08 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`442`, [Linux], :label:`build-fail`: psutil won't compile on kernels < + 2.6.36 due to missing :manpage:`prlimit(2)` syscall. + +1.1.0 — 2013-09-28 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`412`, [Linux]: add :meth:`Process.rlimit`. + +**API changes** + +- :gh:`408`: turn ``STATUS_*`` and ``CONN_*`` constants into plain Python + strings. + +**Performance** + +- :gh:`415`, [Windows]: :meth:`Process.children` is an order of magnitude + faster. +- :gh:`426`, [Windows]: :meth:`Process.name` is an order of magnitude faster. +- :gh:`431`, [POSIX]: :meth:`Process.name` is slightly faster because it + unnecessarily retrieved also :meth:`Process.cmdline`. + +**Build and packaging** + +- :gh:`410`: host tar.gz and Windows binary files are on PyPI. + +**Bug fixes** + +- :gh:`413`, [Windows], :label:`memleak`: :meth:`Process.memory_info` leaks + memory. +- :gh:`392`, [Windows]: :func:`cpu_times_percent` returns negative percentages. +- :gh:`408`: ``STATUS_*`` and ``CONN_*`` constants don't properly serialize on + JSON. +- :gh:`411`, [Windows]: :src:`scripts/disk_usage.py` may pop-up a GUI error. +- :gh:`414`, [Windows]: :meth:`Process.exe` on Windows XP may raise + ``ERROR_INVALID_PARAMETER``. +- :gh:`416`: :func:`disk_usage` doesn't work well with unicode path names. +- :gh:`430`, [Linux]: :meth:`Process.io_counters` report wrong number of r/w + syscalls. +- :gh:`435`, [Linux]: :func:`net_io_counters` might report erreneous NIC names. +- :gh:`436`, [Linux]: :func:`net_io_counters` reports a wrong ``dropin`` value. + +1.0.1 — 2013-07-12 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`405`: :func:`net_io_counters` ``pernic=True`` no longer works as + intended in 1.0.0. + +1.0.0 — 2013-07-10 +^^^^^^^^^^^^^^^^^^ + +**New platforms** + +- :gh:`18`, [SunOS]: add Solaris support (yay!) (thanks Justin Venus) + +**API changes** + +- :gh:`367`: :meth:`Process.connections` :field:`status` is no longer a string + but a constant object (``psutil.CONN_*``). +- :meth:`Process.connections` :field:`local_address` and + :field:`remote_address` fields renamed to :field:`laddr` and :field:`raddr`. +- psutil.network_io_counters() renamed to :func:`net_io_counters`. + +**Internals** + +- :gh:`380`: test suite exits with non-zero on failure. (patch by floppymaster) +- :gh:`391`: introduce unittest2 facilities and provide workarounds if + unittest2 is not installed (Python < 2.7). + +**Bug fixes** + +- :gh:`404`, [Linux], :label:`build-fail`: ``sched_*affinity()`` are implicitly + declared. (patch by Arfrever) +- :gh:`374`, [Windows]: negative memory usage reported if process uses a lot of + memory. +- :gh:`379`, [Linux]: :meth:`Process.memory_maps` may raise :exc:`ValueError`. +- :gh:`394`, [macOS]: :term:`mapped memory` regions of + :meth:`Process.memory_maps` report incorrect file name. + +0.7.1 — 2013-05-03 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`325`, [BSD]: :func:`virtual_memory` can raise :exc:`SystemError`. (patch + by :user:`Jan Beich `) +- :gh:`370`, [BSD]: :meth:`Process.connections` requires root. (patch by + :user:`John Baldwin `) +- :gh:`372`, [BSD]: different process methods raise :exc:`NoSuchProcess` + instead of :exc:`AccessDenied`. + +0.7.0 — 2013-04-12 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`328`, [Windows]: :meth:`Process.ionice` support. +- :gh:`359`: add :func:`boot_time` as a substitute of ``psutil.BOOT_TIME`` + since the latter cannot reflect system clock updates. +- :gh:`361`, [Linux]: :func:`cpu_times` now includes new :field:`steal`, + :field:`guest` and :field:`guest_nice` fields available on recent Linux + kernels. Also, :func:`cpu_percent` is more accurate. +- :gh:`362`: add :func:`cpu_times_percent` (per-CPU-time utilization as a + percentage). + +**API changes** + +- :gh:`246`: psutil.error module is deprecated and scheduled for removal. + +**Internals** + +- :gh:`233`: code migrated to Mercurial (yay!) + +**Bug fixes** + +- :gh:`313`, [Linux], :label:`critical`: :func:`virtual_memory` and + :func:`swap_memory` can crash on certain exotic Linux flavors having an + incomplete ``/proc`` interface. If that's the case we now set the + unretrievable stats to ``0`` and raise :exc:`RuntimeWarning` instead. +- :gh:`341`, [Linux], :label:`critical`: psutil might crash on import due to + error in retrieving system terminals map. +- :gh:`333`, [macOS], :label:`memleak`: leak of Mach ports (patch by + :user:`rsesek`) +- :gh:`339`, [FreeBSD], :label:`memleak`: ``get_pid_list()`` can allocate all + the memory on system. +- :gh:`234`, [Windows]: :func:`disk_io_counters` fails to list certain disks. +- :gh:`264`, [Windows]: use of :func:`disk_partitions` may cause a message box + to appear. +- :gh:`315`, [macOS]: fix some compilation warnings. +- :gh:`317`, [Windows]: cannot set process :term:`CPU affinity` above 31 cores. +- :gh:`319`, [Linux]: :meth:`Process.memory_maps` raises :exc:`KeyError` + 'Anonymous' on Debian squeeze. +- :gh:`321`, [POSIX]: :meth:`Process.ppid` property is no longer cached as the + kernel may set the PPID to 1 in case of a :term:`zombie process`. +- :gh:`323`, [macOS]: :func:`disk_io_counters` ``read_time`` and ``write_time`` + parameters were reporting microseconds not milliseconds. (patch by + :user:`Gregory Szorc `) +- :gh:`331`: :meth:`Process.cmdline` is no longer cached after first access as + it may change. +- :gh:`337`, [Linux]: :class:`Process` methods not working because of a poor + ``/proc`` implementation will raise :exc:`NotImplementedError` rather than + :exc:`RuntimeError` and :meth:`Process.as_dict` will not blow up. (patch by + Curtin1060) +- :gh:`338`, [Linux]: :func:`disk_io_counters` fails to find some disks. +- :gh:`344`, [FreeBSD]: :func:`swap_memory` might return incorrect results due + to ``kvm_open(3)`` not being called. (patch by + :user:`Jean Sebastien `) +- :gh:`351`, [Windows]: if psutil is compiled with MinGW32 (provided installers + for py2.4 and py2.5 are) :func:`disk_io_counters` will fail. (Patch by + m.malycha) +- :gh:`353`, [macOS]: :func:`users` returns an empty list on macOS 10.8. +- :gh:`356`: :meth:`Process.parent` now checks whether parent PID has been + reused in which case returns ``None``. +- :gh:`365`: :meth:`Process.nice` (set) should check PID has not been reused by + another process. +- :gh:`366`, [FreeBSD]: :meth:`Process.memory_maps`, :meth:`Process.num_fds`, + :meth:`Process.open_files` and :meth:`Process.cwd` methods raise + :exc:`RuntimeError` instead of :exc:`AccessDenied`. + +0.6.1 — 2012-08-16 +^^^^^^^^^^^^^^^^^^ + +**API changes** + +- :gh:`316`: :meth:`Process.cmdline` property now makes a better job at + guessing the process executable from the cmdline. +- :meth:`Process.exe` can now return an empty string instead of raising + :exc:`AccessDenied`. + +**Bug fixes** + +- :gh:`316`: :meth:`Process.exe` was resolved in case it was a symlink. +- :gh:`318`: Python 2.4 compatibility was broken. + +0.6.0 — 2012-08-13 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`216`, [POSIX]: add :meth:`Process.connections` UNIX sockets support. +- :gh:`222`, [macOS]: add support for :meth:`Process.cwd`. +- :gh:`261`: add ``Process.memory_info_ex()``. +- :gh:`302`: add :meth:`Process.num_ctx_switches`. +- :gh:`311`: add :func:`virtual_memory` and :func:`swap_memory`. Old + memory-related functions are deprecated. New example scripts: + :src:`scripts/free.py` and :src:`scripts/meminfo.py`. +- :gh:`312`: :func:`net_io_counters` adds 4 new fields: :field:`errin`, + :field:`errout`, :field:`dropin` and :field:`dropout`. + +**API changes** + +- :gh:`295`, [macOS]: :meth:`Process.exe` path is now determined by asking the + OS instead of being guessed from :meth:`Process.cmdline`. +- :gh:`297`, [macOS]: :meth:`Process.name`, :meth:`Process.memory_info`, + :meth:`Process.memory_percent`, :meth:`Process.cpu_times`, + :meth:`Process.cpu_percent`, :meth:`Process.num_threads` no longer raise + :exc:`AccessDenied` for other users' processes and are **2.5x faster**. +- :gh:`301`: :func:`process_iter` now yields processes sorted by their PIDs. +- :gh:`304`, [Windows]: :meth:`Process.create_time`, :meth:`Process.cpu_times`, + :meth:`Process.cpu_percent`, :meth:`Process.memory_info`, + :meth:`Process.memory_percent`, :meth:`Process.num_handles`, + :meth:`Process.io_counters` no longer raise :exc:`AccessDenied` for other + users' processes. +- ``psutil.phymem_usage()`` is deprecated (use :func:`virtual_memory`) +- ``psutil.virtmem_usage()`` is deprecated (use :func:`swap_memory`) +- [Linux]: ``psutil.phymem_buffers()`` is deprecated (use + :func:`virtual_memory`) +- [Linux]: ``psutil.cached_phymem()`` is deprecated (use + :func:`virtual_memory`) + +**Performance** + +- :gh:`220`, [FreeBSD]: :func:`net_connections` has been rewritten in C and no + longer requires ``lsof``. + +**Internals** + +- :gh:`300`: add :src:`scripts/pmap.py`. +- :commit:`6e45ac1a`: add :src:`scripts/netstat.py`. + +**Bug fixes** + +- :gh:`303`, [Windows], :label:`critical`: potential heap corruption in + :meth:`Process.num_threads` and :meth:`Process.status` methods. +- :gh:`306`, :label:`critical`: at C level, errors are not checked when + invoking ``Py*`` functions which create or manipulate Python objects leading + to potential memory related errors and/or segmentation faults. +- :gh:`305`, [FreeBSD], :label:`build-fail`: can't compile on FreeBSD 9 due to + removal of ``utmp.h``. +- :gh:`298`, [macOS], [BSD], :label:`memleak`: memory leak in + :meth:`Process.num_fds`. +- :gh:`299`, :label:`memleak`: potential memory leak every time + ``PyList_New(0)`` is used. +- :gh:`307`, [FreeBSD]: values returned by :func:`net_io_counters` are wrong. +- :gh:`308`, [BSD], [Windows]: ``psutil.virtmem_usage()`` wasn't actually + returning information about :term:`swap memory` usage as it was supposed to + do. It does now. +- :gh:`310`: :meth:`Process.open_files` might not return files which can not be + accessed due to limited permissions. :exc:`AccessDenied` is now raised + instead. + +0.5.1 — 2012-06-29 +^^^^^^^^^^^^^^^^^^ + +**API changes** + +- :gh:`293`, [Windows]: :meth:`Process.exe` path is now determined by asking + the OS instead of being guessed from :meth:`Process.cmdline`. + +**Bug fixes** + +- :gh:`292`, [Linux]: race condition in process :meth:`Process.open_files`, + :meth:`Process.connections`, :meth:`Process.threads`. +- :gh:`294`, [Windows]: :meth:`Process.cpu_affinity` is only able to set CPU + #0. + +0.5.0 — 2012-06-27 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`195`, [Windows]: add :meth:`Process.num_handles`. +- :gh:`209`: :func:`disk_partitions` now provides also mount options. +- :gh:`229`: add :func:`users`. +- :gh:`238`, [Linux], [Windows]: add :meth:`Process.cpu_affinity`. +- :gh:`242`: add ``recursive=True`` to :meth:`Process.children`: return all + process descendants. +- :gh:`260`: add :meth:`Process.memory_maps`. (Windows patch by :user:`wj32`, + macOS patch by :user:`Jeremy Whitlock `) +- :gh:`278`: add :meth:`Process.as_dict`. +- :gh:`284`, [POSIX]: add :meth:`Process.num_fds`. + +**API changes** + +- :gh:`273`: ``psutil.get_process_list()`` is deprecated. +- :gh:`281`: :meth:`Process.ppid`, :meth:`Process.name`, :meth:`Process.exe`, + :meth:`Process.cmdline` and :meth:`Process.create_time` are now cached after + first access, so :exc:`NoSuchProcess` is no longer raised if the process is + gone in the meantime. +- :gh:`282`: ``psutil.STATUS_*`` constants can now be compared by using their + string representation. +- :gh:`290`: :meth:`Process.nice` property is deprecated in favor of new + ``get_nice()`` and ``set_nice()`` methods. + +**Performance** + +- :gh:`245`, [POSIX]: :meth:`Process.wait` incrementally consumes less CPU + cycles. +- :gh:`258`, [Linux]: :meth:`Process.memory_info` is now **0.5x faster**. +- :gh:`262`, [Windows]: :func:`disk_partitions` was slow due to inspecting the + floppy disk drive also when parameter is ``all=False``. +- :gh:`283`: speedup :meth:`Process.is_running` by caching its return value in + case the process is terminated. +- :gh:`287`: :func:`process_iter` now caches :class:`Process` instances between + calls. + +**Build and packaging** + +- :gh:`274`: psutil no longer requires ``2to3`` at installation time in order + to work with Python 3. + +**Dropped support** + +- :gh:`257`, [Windows], :label:`breaking`: removed Windows 2000 support. + +**Bug fixes** + +- :gh:`240`, [macOS], :label:`critical`: incorrect use of ``free()`` for + :meth:`Process.connections`. +- :gh:`193`: :class:`Popen` constructor can throw an exception if the spawned + process terminates quickly. +- :gh:`244`, [POSIX]: :meth:`Process.wait` can hog CPU resources if called + against a process which is not our children. +- :gh:`248`, [Linux]: :func:`net_io_counters` might return erroneous NIC names. +- :gh:`252`, [Windows]: :meth:`Process.cwd` erroneously raise + :exc:`NoSuchProcess` for processes owned by another user. It now raises + :exc:`AccessDenied` instead. +- :gh:`266`, [Windows]: ``psutil.get_pid_list()`` only shows 1024 processes. + (patch by :user:`amoser`) +- :gh:`267`, [macOS]: :meth:`Process.connections` returns wrong remote address. + (Patch by Amoser) +- :gh:`272`, [Linux]: :meth:`Process.open_files` potential race condition can + lead to unexpected :exc:`NoSuchProcess` exception. Also, we can get incorrect + reports of not absolutized path names. +- :gh:`275`, [Linux]: :meth:`Process.io_counters` erroneously raise + :exc:`NoSuchProcess` on old Linux versions. Where not available it now raises + :exc:`NotImplementedError`. +- :gh:`286`: :meth:`Process.is_running` doesn't actually check whether PID has + been reused. +- :gh:`314`: :meth:`Process.children` can sometimes return non-children. + +0.4.1 — 2011-12-14 +^^^^^^^^^^^^^^^^^^ + +**Bug fixes** + +- :gh:`230`, [Windows], [macOS], :label:`memleak`: fix memory leak in + :meth:`Process.connections`. +- :gh:`236`, [Windows], :label:`memleak`: fix memory/handle leak in + :meth:`Process.memory_info`, :meth:`Process.suspend` and + :meth:`Process.resume` methods. +- :gh:`228`: some example scripts were not working with Python 3. +- :gh:`232`, [Linux]: ``psutil.phymem_usage()`` can report erroneous values + which are different than ``free`` command. + +0.4.0 — 2011-10-29 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`150`: add :func:`net_io_counters` (macOS and Windows patch by + :user:`Jeremy Whitlock `) +- :gh:`154`, [FreeBSD]: add support for :meth:`Process.cwd`. +- :gh:`206`: add :func:`disk_io_counters`). (macOS and Windows patch by + :user:`Jeremy Whitlock `) +- :gh:`217`: :meth:`Process.connections` now has a *kind* argument to filter + for connections with different criteria. + +**API changes** + +- :gh:`198`: :meth:`Process.wait` with ``timeout=0`` can now be used to make + the function return immediately. + +**Performance** + +- :gh:`221`, [FreeBSD]: :meth:`Process.open_files` has been rewritten in C and + no longer relies on ``lsof``. + +**Build and packaging** + +- :gh:`157`, [Windows]: provide installer for Python 3.2 64-bit. + +**Internals** + +- :gh:`213`: add :src:`scripts/iotop.py`. +- :gh:`223`: add :src:`scripts/top.py`. +- :gh:`227`: add :src:`scripts/nettop.py`. + +**Bug fixes** + +- :gh:`188`, [Linux], :label:`critical`: psutil import error on Linux ARM + architectures. +- :gh:`200`, [Linux], :label:`critical`: ``psutil.NUM_CPUS`` not working on + armel and sparc architectures and causing crash on module import. +- :gh:`218`, [Linux], :label:`critical`: crash at import time on Debian 64-bit + because of a missing line in :proc:`/proc/meminfo`. +- :gh:`226`, [FreeBSD], :label:`critical`: crash at import time on FreeBSD 7 + and minor. +- :gh:`135`, [macOS]: psutil cannot create :class:`Process` object. +- :gh:`144`, [Linux]: no longer support 0 special PID. +- :gh:`194`, [POSIX]: :meth:`Process.cpu_percent` now reports a percentage over + 100 on multi core processors. +- :gh:`197`, [Linux]: :meth:`Process.connections` is broken on platforms not + supporting IPv6. +- :gh:`201`, [Linux]: :meth:`Process.connections` is broken on big-endian + architectures. +- :gh:`211`: :class:`Process` instance can unexpectedly raise + :exc:`NoSuchProcess` if tested for equality with another object. + +0.3.0 — 2011-07-08 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`125`: add :func:`cpu_times` and :func:`cpu_percent` per-cpu support. +- :gh:`163`: add :meth:`Process.terminal`. +- :gh:`171`: add ``get_phymem()`` and ``get_virtmem()``. Old ``total_*``, + ``avail_*`` and ``used_*`` memory functions are deprecated. +- :gh:`172`: add :func:`disk_usage`. +- :gh:`174`: add :func:`disk_partitions`. + +**Build and packaging** + +- :gh:`179`: setuptools is now used in setup.py + +**Bug fixes** + +- :gh:`159`, [Windows], :label:`memleak`: ``SetSeDebug()`` does not close + handles or unset impersonation on return. +- :gh:`166`, :label:`memleak`: :meth:`Process.memory_info` leaks handles + hogging system resources. +- :gh:`178`, [macOS], :label:`memleak`: :meth:`Process.threads` leaks memory. +- :gh:`164`, [Windows]: :meth:`Process.wait` raises a ``TimeoutException`` when + a process returns ``-1``. +- :gh:`165`: :meth:`Process.status` raises an unhandled exception. +- :gh:`168`: :func:`cpu_percent` returns erroneous results when used in + non-blocking mode. (patch by :user:`Philip Roberts `) +- :gh:`180`, [Windows]: :meth:`Process.num_threads` and :meth:`Process.threads` + methods can raise :exc:`NoSuchProcess` exception while process still exists. + +0.2.1 — 2011-03-20 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`64`: add :meth:`Process.io_counters`. +- :gh:`116`: add :meth:`Process.wait`. +- :gh:`134`: add :meth:`Process.threads`. +- :gh:`137`: add :meth:`Process.uids` and :meth:`Process.gids`. +- :gh:`140`: add :func:`boot_time`. +- :gh:`142`: add :meth:`Process.nice`. +- :gh:`143`: add :meth:`Process.status`. +- :gh:`147`, [Linux]: add :meth:`Process.ionice`. +- :gh:`148`: add :class:`Popen` class combining :class:`subprocess.Popen` and + :class:`Process` in a single interface. + +**API changes** + +- :gh:`136`, [FreeBSD]: :meth:`Process.exe` path is now determined by asking + the kernel instead of guessing it from cmdline[0]. +- :class:`Process` ``uid`` and ``gid`` properties are deprecated in favor of + ``uids`` and ``gids`` properties. + +**Performance** + +- :gh:`152`, [macOS]: :meth:`Process.open_files` rewritten in C (no longer + relies on ``lsof``, **3x faster**). +- :gh:`153`, [macOS]: :meth:`Process.connections` rewritten in C (no longer + relies on ``lsof``, **3x faster**). + +**Bug fixes** + +- :gh:`83`, [macOS]: :meth:`Process.cmdline` is empty on macOS 64-bit. +- :gh:`130`, [Linux]: a race condition can cause :exc:`IOError` exception be + raised on if process disappears between ``open()`` and the subsequent + ``read()`` call. +- :gh:`145`, [Windows]: :exc:`WindowsError` was raised instead of + :exc:`AccessDenied` when using :meth:`Process.resume` or + :meth:`Process.suspend`. +- :gh:`146`, [Linux]: :meth:`Process.exe` property can raise :exc:`TypeError` + if path contains NULL bytes. +- :gh:`151`, [Linux]: :meth:`Process.exe` and :meth:`Process.cwd` for PID 0 + return inconsistent data. + +0.2.0 — 2010-11-13 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`79`: add :meth:`Process.open_files`. +- :gh:`88`: total system physical cached memory. +- :gh:`88`: total system physical memory :term:`buffers` used by the kernel. +- :gh:`91`: add :meth:`Process.send_signal` and :meth:`Process.terminate` + methods. +- :gh:`95`: :exc:`NoSuchProcess` and :exc:`AccessDenied` exception classes now + provide ``pid``, ``name`` and ``msg`` attributes. +- :gh:`97`: add :meth:`Process.children`. +- :gh:`103`: add :meth:`Process.connections`. +- :gh:`111`: add :meth:`Process.exe`. +- :gh:`123`: :func:`cpu_percent` and :meth:`Process.cpu_percent` accept a new + *interval* parameter. +- :gh:`129`: add :meth:`Process.threads`. + +**New platforms** + +- :gh:`107`, [Windows]: add support for Windows 64 bit. (patch by cjgohlke) +- :gh:`117`, [Windows]: added support for Windows 2000. + +**API changes** + +- :gh:`98`: :meth:`Process.cpu_times` and :meth:`Process.memory_info` now + return a named tuple instead of a tuple. +- :gh:`113`: exception messages now include :meth:`Process.name` and + :attr:`Process.pid`. +- ``psutil.Process.path`` property is deprecated and works as an alias for + ``psutil.Process.exe`` property. +- :meth:`Process.kill`: *signal* argument was removed - to send a signal to the + process use :meth:`Process.send_signal` method instead. +- :func:`cpu_times` returns a named tuple instead of a tuple. +- :meth:`Process.cpu_percent` and :func:`cpu_percent` no longer returns + immediately by default (see :gh:`123`). + +**Performance** + +- :gh:`114`, [Windows]: :meth:`Process.username` rewritten in C (no longer uses + WMI, much faster, pywin32 no longer required). (patch by :user:`wj32`) + +**Bug fixes** + +- :gh:`81`, [Windows], :label:`build-fail`: psutil fails to compile with Visual + Studio. +- :gh:`86`, [FreeBSD], :label:`build-fail`: psutil didn't compile against + FreeBSD 6.x. +- :gh:`102`, [Windows], :label:`memleak`: orphaned process handles obtained by + using ``OpenProcess`` in C were left behind every time :class:`Process` class + was instantiated. +- :gh:`80`: fixed warnings when installing psutil with easy_install. +- :gh:`94`: :meth:`Process.suspend` raises :exc:`OSError` instead of + :exc:`AccessDenied`. +- :gh:`111`, [POSIX]: ``path`` and ``name`` :class:`Process` properties report + truncated or erroneous values on POSIX. +- :gh:`120`, [macOS]: :func:`cpu_percent` always returning 100%. +- :gh:`112`: ``uid`` and ``gid`` properties don't change if process changes + effective user/group id at some point. +- :gh:`126`: :meth:`Process.ppid`, :meth:`Process.uids`, :meth:`Process.gids`, + :meth:`Process.name`, :meth:`Process.exe`, :meth:`Process.cmdline` and + :meth:`Process.create_time` properties are no longer cached and correctly + raise :exc:`NoSuchProcess` exception if the process disappears. + +0.1.3 — 2010-03-02 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`14`: :meth:`Process.username`. +- :gh:`51`, [Linux], [Windows]: add :meth:`Process.cwd`. +- :gh:`71`: add :meth:`Process.suspend` and :meth:`Process.resume`. + +**New platforms** + +- :gh:`61`, [FreeBSD]: added support for FreeBSD 64 bit. +- :gh:`75`: Python 3 support. + +**Performance** + +- :gh:`59`: :meth:`Process.is_running` is now **10 times faster**. + +**Bug fixes** + +- :gh:`49`, [FreeBSD], :label:`memleak`: possible memory leak due to missing + ``free()`` on error condition in ``getcmdpath()``. +- :gh:`62`, [Windows], :label:`memleak`: :meth:`Process.cwd` leaked a string + object on every call. +- :gh:`36`: :meth:`Process.cpu_times` and :meth:`Process.memory_info` functions + succeeded. also for dead processes while a :exc:`NoSuchProcess` exception is + supposed to be raised. +- :gh:`48`, [FreeBSD]: incorrect size for MIB array defined in ``getcmdargs``. +- :gh:`50`, [macOS]: fixed ``getcmdargs()`` memory fragmentation. +- :gh:`55`, [Windows]: ``test_pid_4`` was failing on Windows Vista. +- :gh:`57`: some unit tests were failing on systems where no swap memory is + available. +- :gh:`58`: :meth:`Process.is_running` is now called before + :meth:`Process.kill` to make sure we are going to kill the correct process. +- :gh:`73`, [macOS]: virtual memory size reported on includes shared library + size. +- :gh:`77`: :exc:`NoSuchProcess` wasn't raised on :meth:`Process.create_time` + if :meth:`Process.kill` was used first. + +0.1.2 — 2009-05-06 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`32`: add :meth:`Process.cpu_times`. +- :gh:`33`: add :meth:`Process.create_time`. +- :gh:`34`: add :meth:`Process.cpu_percent`. +- :gh:`38`: add :meth:`Process.memory_info`. +- :gh:`41`: add :meth:`Process.memory_percent`. +- :gh:`39`: add :func:`boot_time`. +- :gh:`43`: Total system virtual memory. +- :gh:`46`: Total system physical memory. +- :gh:`44`: Total system used/free virtual and physical memory. + +**Bug fixes** + +- :gh:`36`, [Windows]: :exc:`NoSuchProcess` not raised when accessing timing + methods. +- :gh:`40`, [FreeBSD], [macOS]: fix ``test_get_cpu_times`` failures. +- :gh:`42`, [Windows]: :meth:`Process.memory_percent` raises + :exc:`AccessDenied`. + +0.1.1 — 2009-03-06 +^^^^^^^^^^^^^^^^^^ + +**New APIs** + +- :gh:`9`, [macOS], [Windows]: add ``Process.uid`` and ``Process.gid``, + returning process UID and GID. +- :gh:`11`: per-process parent object: :meth:`Process.parent` property returns + a :class:`Process` object representing the parent process, and + :meth:`Process.ppid` returns the parent PID. +- :gh:`21`, [Windows]: :exc:`AccessDenied` exception created for raising access + denied errors from :exc:`OSError` or :exc:`WindowsError` on individual + platforms. +- :gh:`26`: :func:`process_iter` function to iterate over processes as + :class:`Process` objects with a generator. + +**New platforms** + +- :gh:`4`, [FreeBSD]: support for all functions of psutil. + +**API changes** + +- :gh:`12`, :gh:`15`: :exc:`NoSuchProcess` exception now raised when creating + an object for a nonexistent process, or when retrieving information about a + process that has gone away. +- :class:`Process` objects can now also be compared with == operator for + equality (PID, name, command line are compared). + +**Bug fixes** + +- :gh:`16`, [Windows]: Special case for "System Idle Process" (PID 0) which + otherwise would return an "invalid parameter" exception. +- :gh:`17`: ``get_process_list()`` ignores :exc:`NoSuchProcess` and + :exc:`AccessDenied` exceptions during building of the list. +- :gh:`22`, [Windows]: :meth:`Process.kill` for PID 0 was failing with an unset + exception. +- :gh:`23`, [Linux], [macOS]: create special case for :func:`pid_exists` with + PID 0. +- :gh:`24`, [Windows]: :meth:`Process.kill` for PID 0 now raises + :exc:`AccessDenied` exception instead of :exc:`WindowsError`. +- :gh:`30`: psutil.get_pid_list() was returning two 0 PIDs. + +0.1.0 — 2009-01-27 +^^^^^^^^^^^^^^^^^^ + +Initial release. Supports Linux, Windows, and macOS via per-platform backends +(``_pslinux``, ``_psmswindows``, ``_psosx``) with C extensions for Windows and +macOS. + +**New APIs** + +- :class:`Process` class exposing ``pid``, ``name``, ``path``, and ``cmdline``, + with a ``kill()`` method. +- ``get_process_list()`` returning all running processes. +- ``ProcessInfo`` value object passed between the public API and the platform + backends. diff --git a/docs/conf.py b/docs/conf.py index 9fa163b65e..98617c8e88 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,248 +1,327 @@ -# -*- coding: utf-8 -*- -# -# psutil documentation build configuration file, created by -# sphinx-quickstart. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. +# Copyright (c) 2009, Giampaolo Rodola. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Sphinx configuration file. + +Sphinx doc: +https://www.sphinx-doc.org/en/master/usage/configuration.html +""" import datetime +import importlib.util +import json +import locale import os +import pathlib +import sys +import time + +_HERE = pathlib.Path(__file__).resolve().parent +_ROOT_DIR = _HERE.parent +sys.path.insert(0, str(_HERE / "_ext")) # needed to load local extensions + +# Load _bootstrap.py (at the repo root) without putting the repo +# root on sys.path. Doing so would expose the uncompiled source +# `psutil/` package and shadow any installed psutil, breaking +# `import psutil` at build time (needed by sphinx-codeautolink to +# resolve things like `p.name()` in code blocks). +def _load(path): + spec = importlib.util.spec_from_file_location(path.stem, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +get_version = _load(_ROOT_DIR / "_bootstrap.py").get_version PROJECT_NAME = "psutil" -AUTHOR = "Giampaolo Rodola'" +AUTHOR = "Giampaolo Rodola" THIS_YEAR = str(datetime.datetime.now().year) -HERE = os.path.abspath(os.path.dirname(__file__)) - - -def get_version(): - INIT = os.path.abspath(os.path.join(HERE, '../psutil/__init__.py')) - with open(INIT, 'r') as f: - for line in f: - if line.startswith('__version__'): - ret = eval(line.strip().split(' = ')[1]) - assert ret.count('.') == 2, ret - for num in ret.split('.'): - assert num.isdigit(), ret - return ret - else: - raise ValueError("couldn't find version string") - VERSION = get_version() -# If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.0' +# ===================================================================== +# Core +# ===================================================================== + +needs_sphinx = "9.1" +language = "en" +nitpicky = True # always warn on unresolved cross-references + +# ===================================================================== +# Extensions +# ===================================================================== + +_third_party_exts = [ + "ablog", + "notfound.extension", # custom 404 page + "sphinx.ext.extlinks", + "sphinx.ext.githubpages", # writes .nojekyll + CNAME from html_baseurl + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", + "sphinx_codeautolink", + "sphinx_copybutton", + "sphinx_design", # tabbed code examples on the home page + "sphinx_sitemap", + "sphinxext.opengraph", +] -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = ['sphinx.ext.autodoc', - 'sphinx.ext.coverage', - 'sphinx.ext.pngmath', - 'sphinx.ext.viewcode', - 'sphinx.ext.intersphinx'] +_local_exts = [ # defined in the _ext/ folder + "ablog_extras", + "availability", + "changelog_anchors", + "check_python_syntax", + "field_role", + "genindex_filter", + "giscus", + "glossary_toc", + "label_role", + "notfound_extras", + "opengraph_override", + "post_banner", + "proc_role", + "substitutions", +] -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_template'] +extensions = _third_party_exts + _local_exts -# The suffix of source filenames. -source_suffix = '.rst' +# ===================================================================== +# Project metadata +# ===================================================================== -# The encoding of source files. -# source_encoding = 'utf-8-sig' +project = PROJECT_NAME +author = AUTHOR +version = release = VERSION +copyright = f"2009-{THIS_YEAR} {AUTHOR}" # shown in the footer -# The master toctree document. -master_doc = 'index' +# ===================================================================== +# Cross-references and external links +# ===================================================================== -# General information about the project. -project = PROJECT_NAME -copyright = '2009-%s, %s' % (THIS_YEAR, AUTHOR) - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = VERSION - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -add_function_parentheses = True -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -autodoc_docstring_signature = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme = 'pydoctheme' -html_theme_options = {'collapsiblesidebar': True} - -# Add any paths that contain custom themes here, relative to this directory. -html_theme_path = ["_themes"] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -html_title = "{project} {version} documentation".format(**locals()) - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = 'logo.png' - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -html_favicon = '_static/favicon.ico' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -html_sidebars = { - 'index': 'indexsidebar.html', - '**': ['globaltoc.html', - 'relations.html', - 'sourcelink.html', - 'searchbox.html'] +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), +} +extlinks = { + "gh": ("https://github.com/giampaolo/psutil/issues/%s", "#%s"), + "pr": ("https://github.com/giampaolo/psutil/pull/%s", "PR-%s"), + "user": ("https://github.com/%s", "@%s"), + "commit": ("https://github.com/giampaolo/psutil/commit/%s", "%s"), + "pypi": ("https://pypi.org/project/psutil/%s/", "%s"), + "bpo": ("https://bugs.python.org/issue%s", "BPO-%s"), + "cpy": ("https://github.com/python/cpython/issues/%s", "cpython/#%s"), + "cpy-pr": ("https://github.com/python/cpython/pull/%s", "cpython/PR-%s"), + "src": ("https://github.com/giampaolo/psutil/blob/master/%s", "%s"), } +manpages_url = "https://manpages.debian.org/{path}" -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = { -# 'index': 'indexcontent.html', -# } +# ===================================================================== +# Paths +# ===================================================================== -# If false, no module index is generated. -html_domain_indices = False +exclude_patterns = ["_build"] +rst_prolog = ".. currentmodule:: psutil\n" # Prepended to every .rst file -# If false, no index is generated. -html_use_index = True +# ===================================================================== +# HTML +# ===================================================================== -# If true, the index is split into individual pages for each letter. -# html_split_index = False +# Canonical site URL. Picked up by Sphinx for +# tags. Reused below by sphinxext-opengraph (og:url), sphinx-sitemap, +# and (via blog_baseurl) ablog's atom feed. +html_baseurl = "https://psutil.io/" -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True +# sphinx-notfound-page: absolute URL prefix for static files and +# nav links on 404.html. +notfound_urls_prefix = "/" -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True +html_title = PROJECT_NAME +html_favicon = "_static/images/favicon.svg" +html_last_updated_fmt = "%Y-%m-%d" # ISO date shown in the footer +html_show_sphinx = False # removes "Created using Sphinx" in the footer +html_show_sourcelink = False # removes "View page source" sidebar link -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True +# Sidebar shows method() instead of Class.method() +toc_object_entries_show_parents = "hide" -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' +# ===================================================================== +# Plugins +# ===================================================================== -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None +copybutton_exclude = ".linenos, .gp" -# Output file base name for HTML help builder. -htmlhelp_basename = '%s-doc' % PROJECT_NAME +# ===================================================================== +# Theming +# ===================================================================== -# -- Options for LaTeX output ------------------------------------------------ +# Custom psutil-sphinx-theme, built on top of Sphinx's `basic` theme. -# The paper size ('letter' or 'a4'). -# latex_paper_size = 'letter' +html_theme = "basic" +html_theme_options = { + "globaltoc_maxdepth": 1, + "globaltoc_collapse": False, + "globaltoc_includehidden": True, +} +html_static_path = ["_static"] +html_extra_path = ["_extra"] # robots.txt, copied verbatim to site root +templates_path = ["_templates"] +pygments_style = "tango" # base palette (overridden by css/code.css) -# The font size ('10pt', '11pt' or '12pt'). -# latex_font_size = '10pt' -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass -# [howto/manual]). -latex_documents = [ - ('index', '%s.tex' % PROJECT_NAME, - '%s documentation' % PROJECT_NAME, AUTHOR), -] +def _css_files(): + css_dir = _HERE / "_static" / "css" + # giscus.css is loaded inside the giscus iframe (see + # _templates/comments.html), never by our own pages. Linking it + # here would make every page fetch its @import from giscus.app. + skip = {"giscus.css"} + files = sorted(p.name for p in css_dir.glob("*.css") if p.name not in skip) + head = ["base.css", "fonts.css", "fontawesome.css", "typography.css"] + tail = ["home.css"] + middle = [f for f in files if f not in head + tail] + return [f"css/{name}" for name in head + middle + tail if name in files] -# The name of an image file (relative to this directory) to place at -# the top of the title page. -# latex_logo = None -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False +html_css_files = _css_files() -# If true, show page references after internal links. -# latex_show_pagerefs = False -# If true, show URL addresses after external links. -# latex_show_urls = False +def _js_files(): + js_dir = _HERE / "_static" / "js" + # blog-comment-counts.js is pulled in by the blog listing template + # only (_templates/ablog/collection.html). + files = sorted( + p.name + for p in js_dir.glob("*.js") + if p.name != "blog-comment-counts.js" + ) + return [(f"js/{name}", {"defer": "defer"}) for name in files] -# Additional stuff for the LaTeX preamble. -# latex_preamble = '' -# Documents to append as an appendix to all manuals. -# latex_appendices = [] +html_js_files = _js_files() -# If false, no module index is generated. -# latex_domain_indices = True +# ===================================================================== +# Version selector +# ===================================================================== +VERSIONS = json.loads((_HERE / "versions.json").read_text(encoding="utf-8")) -# -- Options for manual page output ------------------------------------------ +html_context = { + "versions": VERSIONS["versions"], + "versions_current": VERSIONS["current"], +} -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', PROJECT_NAME, '%s documentation' % PROJECT_NAME, [AUTHOR], 1) +# ===================================================================== +# Blog (ablog package) +# ===================================================================== + +# Force UTC for build-time timestamps so atom feed entries are +# the same across build hosts (CI runs UTC; local devs may not). +os.environ["TZ"] = "UTC" +if hasattr(time, "tzset"): + time.tzset() + +# Fix for ablog, which otherwise formats dates in the local locale. +try: + locale.setlocale(locale.LC_TIME, "C") +except locale.Error: + pass + +# Drives atom feed entry s and s. Same value as html_baseurl +# so feed URLs track canonicals URLs. +blog_baseurl = html_baseurl + +# ===================================================================== +# Comments (giscus) +# ===================================================================== + +giscus_repo = "giampaolo/psutil-blog-comments" +giscus_repo_id = "R_kgDOTfVGLA" +giscus_category = "User Comments" +giscus_category_id = "DIC_kwDOTfVGLM4DBrKC" + +# ===================================================================== +# sphinxext-opengraph +# ===================================================================== + +# sphinxext-opengraph emits + Twitter Card tags +# in every page's , so that URLs shared on social medias render +# as rich preview cards instead of bare links. +ogp_site_url = html_baseurl +ogp_site_name = PROJECT_NAME +ogp_description_length = 160 # Google SERP snippet width + +# The logo shown in the preview. sphinxext-opengraph requires a .png +# file. +_logo = "_static/images/logo-psutil.png" +ogp_social_cards = {"image": _logo, "image_mini": _logo} + +# ===================================================================== +# sphinx-sitemap +# ===================================================================== + +# sphinx-sitemap emits /sitemap.xml listing every built page, +# for search engine discovery. Reads html_baseurl; {link} scheme avoids +# the default {lang}{version} prefix (we don't use either in URLs). +sitemap_url_scheme = "{link}" +sitemap_show_lastmod = True +# dirhtml URLs are directories, so match the dir form (e.g. "search/", +# not "search.html") or these leak into the sitemap. +sitemap_excludes = [ + "search/", + "genindex/", + "py-modindex/", + "404/", + "_modules/*", + "blog/archive/", + "blog/drafts/", + "blog/tag/", + "blog/tag/*", + "blog/category/", + "blog/category/*", + "blog/author/", + "blog/author/*", + "blog/????/", # exclude years ] - -# If true, show URL addresses after external links. -# man_show_urls = False +# Suppress sphinx-sitemap warning (turned into error by +# --fail-on-warning) occurring on CI. +suppress_warnings = ["git.too_shallow"] + +# ===================================================================== +# sphinx-codeautolink +# ===================================================================== + +# Treat all code blocks on the same page as one interpreter session: a +# variable defined in block 1 stays known in block 2. Without this, +# snippets like `>>> p = psutil.Process()` followed by `>>> p.name()` +# in a later block lose the type of `p`. +codeautolink_concat_default = True + +# Seed every block with an implicit `import psutil`, so snippets that +# start mid-session (no explicit import line) still have `psutil.X` +# references resolvable. +codeautolink_global_preface = "import psutil" + +# Print warnings for names it can't resolve. +# codeautolink_warn_on_failed_resolve = True + +# ===================================================================== +# Sphinx setup hook +# ===================================================================== + + +def setup(app): + # sphinx-codeautolink needs `import psutil` to resolve things like + # `p.name()` in code blocks. It imports psutil itself internally, + # but silently passes if it can't, so we do it here to crash + # explicitly. Kept inside setup() (not at module scope) so pytest + # collection of docs/test_docs.py doesn't hit it. + import psutil # noqa: F401 + + # ablog and sphinx-codeautolink synthesize pages (blog/tag/*, + # blog/2025, _modules/*) with no .rst behind them. The footer's + # "Edit on GitHub" / "Updated" links would point at files that + # don't exist. + def set_has_rst_source(app, pagename, templatename, context, doctree): + path = pathlib.Path(app.env.doc2path(pagename)) + context["has_rst_source"] = path.is_file() + + app.connect("html-page-context", set_has_rst_source) diff --git a/docs/credits.rst b/docs/credits.rst new file mode 100644 index 0000000000..a583decf43 --- /dev/null +++ b/docs/credits.rst @@ -0,0 +1,452 @@ +Credits +======= + +I would like to recognize some of the people who have been instrumental in the +development of psutil. I'm sure I'm forgetting someone (feel free to email me) +but here is a short list. + +A big thanks to all of you. + +— Giampaolo Rodola + +Top contributors +---------------- + +* :user:`Giampaolo Rodola `: creator, primary author and long-time + maintainer +* :user:`Jay Loden `: original co-author, initial design and project + bootstrap, initial macOS / Windows / FreeBSD implementations +* :user:`Arnon Yaari `: AIX implementation +* :user:`Landry Breuil `: initial OpenBSD implementation +* :user:`Ryo Onodera ` and :user:`Thomas Klausner <0-wiz-0>`: initial + NetBSD implementation + +Donations +--------- + +The following individuals and organizations have supported psutil development +through donations. + +Companies: + +* `Apivoid`_ *(sponsor)* +* `Canonical Juju`_ +* `Canonical Launchpad`_ +* `Canonical`_ +* `Codecov`_ +* `Indeed Engineering`_ +* `Kubernetes`_ +* `Robusta`_ +* `sansec.io`_ *(sponsor)* +* `Sentry`_ +* `Sourcegraph`_ +* `Tidelift`_ *(sponsor)* + +People: + +* :user:`Alex Laird ` +* Alexander Kaftan +* `Alexey Vazhnov`_ +* Amit Kulkarni +* Andrew Bays +* :user:`Artyom Vancyan ` +* Brett Harris +* :user:`c0m4r` +* Carver Koella +* `Chenyoo Hao`_ +* :user:`CoÅŸkun Deniz ` +* :user:`cybersecgeek` +* :user:`Daniel Widdis ` +* :user:`Eugenio E Breijo ` +* :user:`Evan Allrich ` +* Florian Bruhin +* :user:`great-work-told-is` +* Gyula Ãfra +* HTB Industries +* :user:`inarikami` +* :user:`JeremyGrosser` +* :user:`Johannes Maron ` +* :user:`Jakob P. Liljenberg ` +* :user:`Karthik Kumar ` +* Kahntent +* Kristjan Võrk +* Mahmut Dumlupinar +* Marco Schrank +* Matthew Callow +* Mindview LLC +* :user:`Maximilian Wu ` +* Mehver +* mirko +* Morgan Heijdemann +* Oche Ejembi +* :user:`Ofek Lev ` +* Olivier Grisel +* Pavan Maddamsetti +* `PySimpleGUI`_ +* Peter Friedland +* Praveen Bhamidipati +* Remi Chateauneu +* `roboflow.com`_ +* Rodion Stratov +* Russell Robinson +* :user:`SaÅ¡o Živanović ` +* `scoutapm-sponsorships`_ +* Sigmund Vik +* `trashnothing.com`_ +* Thomas Guettler +* Willem de Groot +* Wompasoft +* :user:`Valeriy Abramov ` +* Григорьев Ðндрей + +Code contributors by year +------------------------- + +.. image:: https://img.shields.io/github/contributors/giampaolo/psutil.svg?label=Total%20contributors&style=flat + :target: https://github.com/giampaolo/psutil/graphs/contributors + :alt: contributors + +2026 +~~~~ + +* :user:`Alex Chen ` - :gh:`2859`, :gh:`1959` +* :user:`Amaan Qureshi ` - :gh:`2770` +* :user:`Anshul Nautiyal ` - :gh:`2858` +* :user:`Arman Luthra ` - :gh:`2695` +* :user:`Bert Pluymers ` - :gh:`2642` +* :user:`Data-hYg ` - :gh:`2789` +* :user:`Ding Qiuran ` - :gh:`2860` +* :user:`Ehtesham Siddiqui ` - :gh:`2793` +* :user:`Felix Yan ` - :gh:`2732` +* :user:`Gabriel Changamire ` - :gh:`2809` +* :user:`Hanson Wang ` - :gh:`2810` +* :user:`Jinhyuk Hong ` - :gh:`2855` +* :user:`Julien Stephan ` - :gh:`2512` +* :user:`Karl Hill ` - :gh:`2857` +* :user:`Kataoka Katsuki ` - :gh:`2854` +* :user:`Marcel Telka ` - :gh:`2687` +* :user:`Omprakash Chauhan ` - :gh:`2655` +* :user:`Robert Kirkman ` - :gh:`2611` +* :user:`Santhosh Raju ` - :gh:`2805` +* :user:`Sebastian Cao ` - :gh:`2628` +* :user:`Sergey Fedorov ` - :gh:`2701` +* :user:`Tobias Klauser ` - :gh:`2711` + +2025 +~~~~ + +* :user:`Ben Peddell ` - :gh:`2495`, :gh:`2568` +* :user:`Ben Raz ` - :gh:`2643` +* :user:`Eli Wenig ` - :gh:`2638` +* :user:`Fabien Bousquet ` - :gh:`2529` +* :user:`Irene Sheen ` - :gh:`2606` +* :user:`Isaac K. Ko <1saac-k>` - :gh:`2612` +* :user:`Jonathan Kohler ` - :gh:`2527` +* :user:`Lysandros Nikolaou ` - :gh:`2565`, :gh:`2588`, + :gh:`2589`, :gh:`2590`, :gh:`2591`, :gh:`2609`, :gh:`2615`, :gh:`2627`, + :gh:`2659` (wheels for free-threaded Python) +* :user:`Marcel Telka ` - :gh:`2469`, :gh:`2545`, :gh:`2546`, + :gh:`2592`, :gh:`2594` +* :user:`Matthieu Darbois ` - :gh:`2503`, :gh:`2581` (Windows ARM64 + wheels) +* :user:`Sergey Fedorov ` - :gh:`2694` +* :user:`Will Hawes ` - :gh:`2496` +* :user:`Xianpeng Shen ` - :gh:`2640` + +2024 +~~~~ + +* :user:`Aleksey Lobanov ` - :gh:`2457` +* :user:`Cristian Vîjdea ` - :gh:`2442` +* :user:`Matthieu Darbois ` - :gh:`2370`, :gh:`2375`, :gh:`2417`, + :gh:`2425`, :gh:`2429`, :gh:`2450`, :gh:`2479`, :gh:`2486` (macOS and Linux + ARM64 wheels) +* :user:`Mayank Jha ` - :gh:`2379` +* :user:`Oliver Tomé ` - :gh:`2222` +* :user:`Ryan Carsten Schmidt ` - :gh:`2361`, :gh:`2364`, + :gh:`2365` +* :user:`Sam Gross ` - :gh:`2401`, :gh:`2402`, :gh:`2427`, + :gh:`2428` (free-threading Python) +* :user:`Shade Gladden ` - :gh:`2376` + +2023 +~~~~ + +* :user:`Amir Rossert ` - :gh:`2346` +* :user:`Matthieu Darbois ` - :gh:`2211`, :gh:`2216`, :gh:`2246`, + :gh:`2247`, :gh:`2252`, :gh:`2269`, :gh:`2270`, :gh:`2315` +* :user:`Po-Chuan Hsieh ` - :gh:`2186`, :gh:`1646` +* :user:`Thomas Klausner <0-wiz-0>` - :gh:`2241` +* :user:`Xuehai Pan ` - :gh:`2266` + +2022 +~~~~ + +* :user:`Amir Rossert ` - :gh:`2156`, :gh:`2345` +* :user:`Bernhard Urban-Forster ` - :gh:`2135` +* :user:`Chris Lalancette ` - :gh:`2037` (:func:`net_if_stats` + flags arg on POSIX) +* :user:`Daniel Li ` - :gh:`2150` +* :user:`Daniel Widdis ` - :gh:`2077`, :gh:`2160` +* :user:`Garrison Carter ` - :gh:`2096` +* :user:`Hiroyuki Tanaka ` - :gh:`2086` +* :user:`Hugo van Kemenade ` - :gh:`2099` (Drop Python 2.6 support) +* :user:`Lawrence D'Anna ` - :gh:`2010` +* :user:`Matthieu Darbois ` - :gh:`1954`, :gh:`2021`, :gh:`2039`, + :gh:`2040`, :gh:`2102`, :gh:`2111`, :gh:`2142`, :gh:`2145`, :gh:`2146`, + :gh:`2147`, :gh:`2153`, :gh:`2155`, :gh:`2168` +* :user:`Steve Dower ` - :gh:`2080` +* :user:`Thomas Klausner <0-wiz-0>` - :gh:`2088`, :gh:`2128` +* :user:`Torsten Blum ` - :gh:`2114` + +2021 +~~~~ + +* :user:`David Knaack ` - :gh:`1921` +* :user:`Guillermo ` - :gh:`1913` +* :user:`Martin LiÅ¡ka ` - :gh:`1851` +* :user:`MaWe2019 ` - :gh:`1953` +* :user:`Nikita Radchenko ` - :gh:`1940` +* :user:`Oleksii Shevchuk ` - :gh:`1904` +* :user:`Olivier Dormond ` - :gh:`1956` +* :user:`Pablo Baeyens ` - :gh:`1598` +* :user:`PetrPospisil ` - :gh:`1980` +* :user:`Saeed Rasooli ` - :gh:`1996` +* :user:`Wilfried Goesgens ` - :gh:`1990` +* :user:`Xuehai Pan ` - :gh:`1949` + +2020 +~~~~ + +* :user:`Anselm Kruis ` - :gh:`1695` +* :user:`Armin Gruner ` - :gh:`1800` (:meth:`Process.environ` on + BSD) +* :user:`Chris Burger ` - :gh:`1830` +* :user:`vser1 ` - :gh:`1637` +* :user:`Grzegorz Bokota ` - :gh:`1758`, :gh:`1762` +* :user:`Jake Omann ` - :gh:`1876` +* :user:`Jakob P. Liljenberg ` - :gh:`1837`, :gh:`1838` +* :user:`Javad Karabi ` - :gh:`1648` +* :user:`Julien Lebot ` - :gh:`1768` (Windows Nano server + support) +* :user:`MichaÅ‚ Górny ` - :gh:`1726` +* :user:`Mike Hommey ` - :gh:`1665` +* :user:`Po-Chuan Hsieh ` - :gh:`1646` +* :user:`Riccardo Schirone ` - :gh:`1616` +* :user:`Tim Schlueter ` - :gh:`1708` +* :user:`Vincent A. Arcila ` - :gh:`1620`, :gh:`1727` + +2019 +~~~~ + +* :user:`qcha0 ` - :gh:`1491` +* :user:`Alex Manuskin ` - :gh:`1487` +* :user:`Ammar Askar ` - :gh:`1485` (:func:`getloadavg` on Windows) +* :user:`Arnon Yaari ` - :gh:`607`, :gh:`1349`, :gh:`1409`, + :gh:`1500`, :gh:`1505`, :gh:`1507`, :gh:`1533` +* :user:`Athos Ribeiro ` - :gh:`1585` +* :user:`Benjamin Drung ` - :gh:`1462` +* :user:`Bernát Gábor ` - :gh:`1565` +* :user:`Cedric Lamoriniere ` - :gh:`1470` +* :user:`Daniel Beer ` - :gh:`1471` +* :user:`David Brochart ` - :gh:`1493`, :gh:`1496` +* :user:`EccoTheFlintstone ` - :gh:`1368`, :gh:`1348` +* :user:`Erwan Le Pape ` - :gh:`1570` +* :user:`Ghislain Le Meur ` - :gh:`1379` +* :user:`Kamil Rytarowski ` - :gh:`1526`, :gh:`1530`, :gh:`1534` + (:meth:`Process.cwd` for NetBSD) +* :user:`Nathan Houghton ` - :gh:`1619` +* :user:`Samer Masterson ` - :gh:`1480` +* :user:`Xiaoling Bao ` - :gh:`1223` +* Mozilla Foundation - Sample code for process :term:`USS` memory + +2018 +~~~~ + +* :user:`Alex Manuskin ` - :gh:`1284`, :gh:`1345`, :gh:`1350`, + :gh:`1369` (:func:`sensors_temperatures` for macOS, FreeBSD, Linux) +* :user:`Arnon Yaari ` - :gh:`1214` +* :user:`Dan Vinakovsky ` - :gh:`1216` +* :user:`Denis Krienbühl ` - :gh:`1260` +* :user:`Ilya Yanok ` - :gh:`1332` +* :user:`janderbrain ` - :gh:`1169` +* :user:`Jaime Fullaondo ` - :gh:`1320` +* :user:`Jean-Luc Migot ` - :gh:`1258`, :gh:`1289` +* :user:`Koen Kooi ` - :gh:`1360` +* :user:`Lawrence Ye ` - :gh:`1321` +* :user:`Maxime Mouial ` - :gh:`1239` +* :user:`Nikhil Marathe ` - :gh:`1278` +* :user:`stswandering ` - :gh:`1243` +* :user:`Sylvain Duchesne ` - :gh:`1294` + +2017 +~~~~ + +* :user:`Adrian Page ` - :gh:`1160` +* :user:`Akos Kiss ` - :gh:`1150` +* :user:`Alexander Hasselhuhn ` - :gh:`1022` +* :user:`Antoine Pitrou ` - :gh:`1186` +* :user:`Arnon Yaari ` - :gh:`1130`, :gh:`1137`, :gh:`1145`, + :gh:`1156`, :gh:`1164`, :gh:`1174`, :gh:`1177`, :gh:`1123` (AIX + implementation) +* :user:`Baruch Siach ` - :gh:`872` +* :user:`Danek Duvall ` - :gh:`1002` +* :user:`Gleb Smirnoff ` - :gh:`1070`, :gh:`1076`, :gh:`1079` +* :user:`Himanshu Shekhar ` - :gh:`1036` +* :user:`Jakub Bacic ` - :gh:`1127` +* :user:`Matthew Long ` - :gh:`1167` +* :user:`Nicolas Hennion ` - :gh:`974` +* :user:`Oleksii Shevchuk ` - :gh:`1091`, :gh:`1093`, :gh:`1220`, + :gh:`1346` +* :user:`Pierre Fersing ` - :gh:`950` +* :user:`Sebastian Saip ` - :gh:`1141` +* :user:`Thiago Borges Abdnur ` - :gh:`959` +* :user:`Yannick Gingras ` - :gh:`1057` + +2016 +~~~~ + +* :user:`Andre Caron ` - :gh:`880` +* :user:`Arcadiy Ivanov ` - :gh:`919` +* :user:`ewedlund ` - :gh:`874` +* :user:`Eric Rahm ` - :gh:`745`, :gh:`746` (:term:`USS` memory on + macOS and Windows) +* :user:`Farhan Khan ` - :gh:`823` +* :user:`Frank Benkstein ` - :gh:`732`, :gh:`733`, :gh:`736`, + :gh:`738`, :gh:`739`, :gh:`740` +* :user:`Ilya Georgievsky ` - :gh:`870` +* :user:`Jake Omann ` - :gh:`816`, :gh:`775`, :gh:`1874` +* :user:`Jeremy Humble ` - :gh:`863` +* :user:`Landry Breuil ` - :gh:`741` +* :user:`Mark Derbecker ` - :gh:`660` +* :user:`Max Bélanger ` - :gh:`936`, :gh:`1133` +* :user:`Patrick Welche ` - :gh:`812` +* :user:`Syohei YOSHIDA ` - :gh:`730` +* :user:`Timmy Konick ` - :gh:`751` +* :user:`Yago Jesus ` - :gh:`798` + +2015 +~~~~ + +* :user:`Arnon Yaari ` - :gh:`680`, :gh:`679`, :gh:`610` +* :user:`Bruno Binet ` - :gh:`572` +* :user:`Denis ` - :gh:`541` +* :user:`Fabian Groffen ` - :gh:`611`, :gh:`618` +* :user:`Gabi Davar ` - :gh:`578`, :gh:`581`, :gh:`587` +* :user:`Jeff Tang ` - :gh:`616`, :gh:`648`, :gh:`653`, :gh:`654` +* :user:`John Burnett ` - :gh:`614` +* :user:`karthik ` - :gh:`568` +* :user:`Landry Breuil ` - :gh:`713`, :gh:`709` (OpenBSD + implementation) +* :user:`Mike Sarahan ` - :gh:`690` +* :user:`Sebastian-Gabriel Brestin ` - :gh:`704` +* :user:`sk6249 ` - :gh:`670` +* :user:`spacewander ` - :gh:`561`, :gh:`603`, :gh:`555` +* :user:`Steven Winfield ` - :gh:`672` +* :user:`Sylvain Mouquet ` - :gh:`565` +* :user:`Ãrni Már Jónsson ` - :gh:`634` +* :user:`Ryo Onodera `: + `e124acba `_ (NetBSD + implementation) + +2014 +~~~~ + +* :user:`Alexander Grothe ` - :gh:`497` +* :user:`Anders Chrigström ` - :gh:`548` +* Francois Charron - :gh:`474` +* :user:`Guido Imperiale ` - :gh:`470`, :gh:`477` +* :user:`Jeff Tang ` - :gh:`340`, :gh:`519`, :gh:`529`, :gh:`654` +* :user:`Marc Abramowitz ` - :gh:`492` +* Naveed Roudsari - :gh:`421` +* :user:`Yaolong Huang ` - :gh:`530` + +2013 +~~~~ + +* :user:`Arfrever.FTA ` - :gh:`404` +* :user:`Daniel Fox ` - :gh:`386` +* Jason Kirtland - backward compatible implementation of + collections.defaultdict +* :user:`John Baldwin ` - :gh:`370` +* John Pankov - :gh:`435` +* :user:`Josiah Carlson ` - :gh:`451`, :gh:`452` +* m.malycha - :gh:`351` +* :user:`Matt Good ` - :gh:`438` +* :user:`Thomas Klausner <0-wiz-0>` - :gh:`557` (NetBSD implementation) +* Ulrich Klank - :gh:`448` + +2012 +~~~~ + +* :user:`Amoser` - :gh:`266`, :gh:`267`, :gh:`340` +* :user:`Florent Xicluna ` - :gh:`319` +* :user:`Gregory Szorc ` - :gh:`323` +* :user:`Jan Beich ` - :gh:`344` +* Youngsik Kim - :gh:`317` + +2011 +~~~~ + +* :user:`Jeremy Whitlock ` - :gh:`125`, :gh:`150`, :gh:`206`, + :gh:`217`, :gh:`260` (:func:`net_io_counters` and :func:`disk_io_counters` on + macOS) + +2010 +~~~~ + +* :user:`Christoph Gohlke ` - :gh:`107` +* :user:`Wen Jia Liu (wj32) ` - :gh:`114`, :gh:`115` + +2009 +~~~~ + +* Yan Raber: `c861c08b `_ + (Windows :func:`cpu_times`), + `15159111 `_ (Windows + :meth:`Process.username`) +* :user:`Jay Loden ` - + `79128baa `_ (first + commit of FreeBSD implementation) + +2008 +~~~~ + +* :user:`Jay Loden ` - + `efe9236a `_ (first + commit of macOS implementation) +* Dave Daeschler - + `71875761 `_ (first + commit of Windows implementation) +* :user:`Giampaolo Rodola ` - + `6296c2ab `_ (first + commit of Linux implementation) +* :user:`Giampaolo Rodola ` - + `8472a17f `_ (inception + / initial directory structure) + +.. People Donors +.. ============================================================================ + +.. _`Alexey Vazhnov`: https://opencollective.com/alexey-vazhnov +.. _`Chenyoo Hao`: https://opencollective.com/chenyoo-hao +.. _`PySimpleGUI`: https://github.com/PySimpleGUI +.. _`roboflow.com`: https://github.com/roboflow +.. _`sansec.io`: https://github.com/sansecio +.. _`scoutapm-sponsorships`: https://github.com/scoutapm-sponsorships +.. _`trashnothing.com`: https://github.com/Trash-Nothing + +.. Company donors +.. ============================================================================ + +.. _`Apivoid`: https://www.apivoid.com +.. _`Canonical Juju`: https://github.com/juju +.. _`Canonical Launchpad`: https://launchpad.net/ +.. _`Canonical`: https://github.com/canonical +.. _`Codecov`: https://github.com/codecov +.. _`Kubernetes`: https://github.com/kubernetes/kubernetes +.. _`Indeed Engineering`: https://github.com/indeedeng +.. _`Robusta`: https://github.com/robusta-dev +.. _`Sentry`: https://sentry.io/ +.. _`Sourcegraph`: https://sourcegraph.com/ +.. _`Tidelift`: https://tidelift.com diff --git a/docs/devguide.rst b/docs/devguide.rst new file mode 100644 index 0000000000..42331c8009 --- /dev/null +++ b/docs/devguide.rst @@ -0,0 +1,221 @@ +Development guide +================= + +.. seealso:: `Contributing to psutil project `_ + +Build, setup and test +--------------------- + +- psutil makes extensive use of C code, so a C compiler and the Python + development headers are required. First clone the repository: + + .. code-block:: bash + + git clone https://github.com/giampaolo/psutil.git + cd psutil + + On Linux, FreeBSD, OpenBSD, NetBSD and Solaris, install the system deps: + + .. code-block:: bash + + make install-sysdeps # compiler + python headers + make install-sysdeps-test # CLI tools used by tests + + On macOS, AIX and Windows there's no such target, see + :ref:`install_from_source`. Then, everywhere: + + .. code-block:: bash + + make install-pydeps-dev # python development deps (linters, etc) + make build # compile the C extension in place + make test + +- ``make`` (via the :src:`Makefile`) is used for building, testing and general + development tasks, including on Windows (see below): + + .. code-block:: bash + + make clean + make test + make test-parallel + make test-memleaks + make coverage + make lint-all + make fix-all + make uninstall + make help + +- To run a specific test: + + .. code-block:: none + + make test ARGS=tests/test_system.py + +- ``make build`` compiles the extension in place, so you can import psutil + straight from the repo. No need to install it. + +- Don't use ``sudo``, except for the ``install-sysdeps-*`` targets, which + invoke it themselves when needed. + +- To target a specific Python version, pass ``PYTHON`` to every step, so that + the extension is built by the same interpreter that runs the tests: + + .. code-block:: none + + make install-pydeps-dev PYTHON=python3.13 + make build PYTHON=python3.13 + make test PYTHON=python3.13 + +Windows +------- + +- The recommended way to develop on Windows is to use ``make``. +- For the build tools, Git Bash and GNU Make setup see :ref:`install_windows`. +- Once inside a Git Bash shell, run: + + .. code-block:: bash + + make install-pydeps-dev + make build + make test-parallel + +.. _devguide_debug_mode: + +Debug mode +---------- + +If you need to debug unusual situations or report a bug, you can enable debug +mode via the :envvar:`PSUTIL_DEBUG` environment variable. In this mode, psutil +may print additional information to stderr. Usually these are non-severe error +conditions that are ignored instead of causing a crash. Unit tests +automatically run with debug mode enabled. To enable debug mode in UNIX (or on +Windows + Bash): + +.. code-block:: none + + $ PSUTIL_DEBUG=1 python3 test_script.py + psutil-debug [psutil/_psutil_linux.c:150]> setmntent() failed (ignored) + +On Windows using cmd.exe: + +.. code-block:: none + + set PSUTIL_DEBUG=1 && python.exe test_script.py + psutil-debug [psutil/arch/windows/proc.c:56]> ReadProcessMemory -> ERROR_NOACCESS (ignored) + +Coding style +------------ + +All style and formatting checks are enforced locally on each ``git commit`` and +via a GitHub Actions pipeline. + +- Python: follows `PEP-8`_, formatted and linted with ``black`` and ``ruff``. +- C: generally follows `PEP-7`_, formatted with ``clang-format``. +- Other files (``.rst``, ``.toml``, ``.md``, ``.yml``): validated by linters. + +The pipeline re-runs all checks for consistency (``make lint-all``). + +Run ``make fix-all`` before committing; it usually fixes Python issues (via +``black`` and ``ruff``) and C issues (via ``clang-format``). + +Code organization +----------------- + +Not every API reaches C: many are implemented in python alone (on Linux, by +parsing ``/proc``). For those that do, a call travels down through the +platform-specific layers. Linux is used here as an example: + +.. code-block:: none + + import psutil + │ + â–¼ + psutil/__init__.py public API, Process class + │ + â–¼ + psutil/_pslinux.py python layer: parses /proc, calls into C + │ + â–¼ + psutil/_psutil_linux.c C extension entry point (arg parsing) + │ + â–¼ + psutil/arch/linux/*.c platform-specific C implementation + + arch/posix/*.c shared by POSIX + + arch/all/*.c shared by everything + +Where things live: + +.. code-block:: bash + + psutil/__init__.py # Public API ("import psutil") + psutil/_common.py # Generic utilities + psutil/_ntuples.py # Named tuples returned by psutil APIs + psutil/_enums.py # Enum containers + psutil/_ps{platform}.py # OS-specific python wrapper + psutil/_psutil_{platform}.c # OS-specific C extension (entry point) + psutil/arch/all/*.c # C code common to all OSes + psutil/arch/posix/*.c # C code common to POSIX OSes + psutil/arch/bsd/*.c # C code common to the BSDs + psutil/arch/{platform}/*.c # OS-specific C implementation + tests/test_process.py # Main process API tests + tests/test_system.py # Main system API tests + tests/test_{platform}.py # OS-specific tests + +Adding a new API +---------------- + +- Define the public API in :src:`psutil/__init__.py`. +- Implement it for each applicable platform in ``psutil/_ps{platform}.py`` + (e.g. :src:`psutil/_pslinux.py`). +- If needed, add C code in ``psutil/arch/{platform}/file.c``. +- Add a generic test in :src:`tests/test_system.py` or + :src:`tests/test_process.py`. +- Add a platform-specific test in ``tests/test_{platform}.py``. +- Update :src:`docs/api.rst`. +- Open a pull request. + +Make a pull request +------------------- + +- Fork psutil on GitHub. +- Clone your fork: ``git clone git@github.com:YOUR-USERNAME/psutil.git`` +- Create a branch: ``git checkout -b new-feature`` +- Stage and commit: ``git add `` then + ``git commit -m 'Add some feature'`` +- Push: ``git push origin new-feature`` +- Open a pull request (see :src:`CONTRIBUTING.md`). + +Continuous integration +---------------------- + +Tests run automatically on pull requests and on relevant pushes, covering all +regularly tested platforms except AIX. See +`.github/workflows `_. + +Documentation +------------- + +- Source is in the :src:`docs/ ` directory. +- To build HTML: + + .. code-block:: bash + + make install-pydeps-docs + cd docs/ + make html + +- The documentation is hosted at https://psutil.io. It's a single version, + rebuilt and deployed automatically on every push to ``master``. + +Releases +-------- + +For project maintainers: + +- Releases are uploaded to `PyPI`_ via ``make release``. +- Git tags use the ``vX.Y.Z`` format (e.g. ``v7.2.2``). +- The version string is defined in :src:`psutil/__init__.py` (``__version__``). + +.. _`PEP-7`: https://www.python.org/dev/peps/pep-0007/ +.. _`PEP-8`: https://www.python.org/dev/peps/pep-0008/ +.. _`PyPI`: https://pypi.org/project/psutil/ diff --git a/docs/faq.rst b/docs/faq.rst new file mode 100644 index 0000000000..162878de79 --- /dev/null +++ b/docs/faq.rst @@ -0,0 +1,402 @@ +FAQ +=== + +This section answers common questions and pitfalls when using psutil. + +General +------- + +.. _faq_named_tuple_unpacking: + +Why should I avoid positional unpacking of named tuples? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Most psutil functions return named tuples. It is tempting to unpack them +positionally, but **field order may change across major releases** (as happened +in 8.0 with :func:`cpu_times` and :meth:`Process.memory_info`). Always use +attribute access instead: + +.. code-block:: python + + # bad + rss, vms = p.memory_info() + + # good + m = p.memory_info() + print(m.rss, m.vms) + +See the :ref:`migration guide ` for the full list of field-order +changes in 8.0. + +Exceptions +---------- + +.. _faq_access_denied: + +Why do I get AccessDenied? +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:exc:`AccessDenied` is raised when the OS refuses to return information about a +process because the calling user does not have sufficient privileges. This is +expected behavior and is not a bug. It typically happens when: + +- querying processes owned by other users (e.g. *root*) +- calling certain methods like :meth:`Process.memory_maps`, + :meth:`Process.open_files` or :meth:`Process.net_connections` for privileged + processes + +You have two options to deal with it. + +- Option 1: call the method directly and catch the exception: + + .. code-block:: python + + import psutil + + p = psutil.Process(pid) + try: + print(p.memory_maps()) + except (psutil.AccessDenied, psutil.NoSuchProcess): + pass + +- Option 2: use :func:`process_iter` with a list of attribute names to + pre-fetch. Both :exc:`AccessDenied` and :exc:`NoSuchProcess` are handled + internally: the corresponding method returns ``None`` (or ``ad_value``) + instead of raising. This also avoids the race condition where a process + disappears between iteration and method call: + + .. code-block:: python + + import psutil + + for p in psutil.process_iter(["name", "username"], ad_value="N/A"): + print(p.name(), p.username()) # no try/except needed + +.. _faq_no_such_process: + +Why do I get NoSuchProcess? +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:exc:`NoSuchProcess` is raised when a process no longer exists. The most common +cause is a race condition: a process can die between the moment its PID is +obtained and the moment it is queried. The following two naive patterns are +racy: + +.. code-block:: python + + import psutil + + for pid in psutil.pids(): + p = psutil.Process(pid) # may raise NoSuchProcess + print(p.name()) # may raise NoSuchProcess + +.. code-block:: python + + import psutil + + if psutil.pid_exists(pid): + p = psutil.Process(pid) # may raise NoSuchProcess + print(p.name()) # may raise NoSuchProcess + +The correct approach is to use :func:`process_iter`, which handles +:exc:`NoSuchProcess` internally and skips processes that disappear during +iteration: + +.. code-block:: python + + import psutil + + for p in psutil.process_iter(["name"]): + print(p.name()) + +If you have a specific PID (e.g. a known child process), wrap the call in a +try/except: + +.. code-block:: python + + import psutil + + try: + p = psutil.Process(pid) + print(p.name(), p.status()) + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + +You can also catch :exc:`Error`, which implies both :exc:`AccessDenied` and +:exc:`NoSuchProcess`: + +.. code-block:: python + + import psutil + + try: + p = psutil.Process(pid) + print(p.name(), p.status()) + except psutil.Error: + pass + +Processes +--------- + +.. _faq_pid_reuse: + +PID reuse +^^^^^^^^^ + +Operating systems recycle PIDs. A :class:`Process` object obtained now may +later refer to a different process if the original one terminated and a new one +was assigned the same PID. + +**How psutil handles this:** + +- *Most read-only methods* (e.g. :meth:`Process.name`, + :meth:`Process.cpu_percent`) do **not** check for PID reuse and instead query + whatever process currently holds that PID. + +- *Signal methods* (e.g. :meth:`Process.send_signal`, :meth:`Process.suspend`, + :meth:`Process.resume`, :meth:`Process.terminate`, :meth:`Process.kill`) + **do** check for PID reuse (via PID + creation time) before acting, raising + :exc:`NoSuchProcess` if the PID was recycled. This prevents accidentally + killing the wrong process (:bpo:`6973`). + +- *Set methods* :meth:`Process.nice` (set), :meth:`Process.ionice` (set), + :meth:`Process.cpu_affinity` (set), and :meth:`Process.rlimit` (set) also + perform this check before applying changes. + +:meth:`Process.is_running` is the recommended way to verify whether a +:class:`Process` instance still refers to the same process. It compares PID and +creation time, and returns ``False`` if the PID was reused. Prefer it over +:func:`pid_exists`. + +.. note:: + + On FreeBSD, OpenBSD, SunOS and AIX the PID reuse check is disabled, and + process identity is based on the PID alone. That's because on these platforms + the process creation time is not stable across system clock updates (e.g. + NTP), which previously caused false :exc:`NoSuchProcess` exceptions for + processes which were still alive (:gh:`2888`). + +.. _faq_zombie_process: + +What is a zombie process? +^^^^^^^^^^^^^^^^^^^^^^^^^ + +A :term:`zombie process` is a process that has finished execution but whose +entry remains in the process table until the parent calls ``wait()``. When +psutil encounters a :term:`zombie process` it raises :exc:`ZombieProcess`, a +subclass of :exc:`NoSuchProcess`. + +**Behavior:** + +- A zombie process can be instantiated via :class:`Process` (pid) without + error. +- :meth:`Process.status` always returns :data:`STATUS_ZOMBIE`. +- :meth:`Process.is_running` and :func:`pid_exists` return ``True``. +- The zombie appears in :func:`process_iter` and :func:`pids`. +- Sending signals (:meth:`Process.terminate`, :meth:`Process.kill`, etc.) has + no effect. +- Most methods (:meth:`Process.cmdline`, :meth:`Process.exe`, + :meth:`Process.memory_maps`, etc.) may raise :exc:`ZombieProcess`, return a + meaningful value, or return a null/empty value depending on the platform. +- :meth:`Process.as_dict` will not crash. + +**How to create a zombie:** + +.. code-block:: python + + import os, time + + pid = os.fork() # the zombie + if pid == 0: + os._exit(0) # child exits immediately + else: + time.sleep(1000) # parent does NOT call wait() + +**How to detect zombies:** + +.. code-block:: python + + import psutil + + for p in psutil.process_iter(["status"]): + if p.status() == psutil.STATUS_ZOMBIE: + print(f"zombie: pid={p.pid}") + +**How to get rid of a zombie:** + +The only way is to have its parent process call ``wait()`` (or ``waitpid()``). +If the parent never does this, killing the parent will cause the zombie to be +re-parented to ``init`` / ``systemd``, which will reap it automatically. + +.. _faq_open_files_windows: + +Why does open_files() not return all files on Windows? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:meth:`Process.open_files` on Windows is not guaranteed to enumerate all +regular file handles. The underlying Windows API may hang when retrieving +certain :term:`handle` names, so psutil spawns a thread to query each handle +and kills it if it doesn't respond within 100 ms. This means some entries can +be missed. This is a known OS-level limitation shared by tools like Process +Hacker (see `issue 597 `_). + +.. _faq_pid_exists_vs_isrunning: + +What is the difference between pid_exists() and Process.is_running()? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:func:`pid_exists` checks whether a PID is present in the process list. +:meth:`Process.is_running` does the same, but also detects +:ref:`PID reuse ` by comparing the process creation time. Use +:func:`pid_exists` when you have a bare PID and don't need to guard against +reuse (it's faster). Use :meth:`Process.is_running` when you hold a +:class:`Process` object and want to confirm it still refers to the same +process. + +CPU +--- + +.. _faq_cpu_percent: + +Why does cpu_percent() return 0.0 on first call? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:func:`cpu_percent` (and :meth:`Process.cpu_percent`) measures CPU usage +*between two calls*. The very first call has no prior sample to compare +against, so it always returns ``0.0``. The fix is to call it once to initialize +the baseline, discard the result, then call it again after a short sleep: + +.. code-block:: python + + import time + import psutil + + psutil.cpu_percent() # discard first call + time.sleep(0.5) + print(psutil.cpu_percent()) # meaningful value + +Alternatively, pass ``interval`` to make it block internally: + +.. code-block:: python + + print(psutil.cpu_percent(interval=0.5)) + +The same applies to :meth:`Process.cpu_percent`: + +.. code-block:: python + + p = psutil.Process() + p.cpu_percent() # discard + time.sleep(0.5) + print(p.cpu_percent()) # meaningful value + +.. _faq_cpu_percent_gt_100: + +Can Process.cpu_percent() return a value higher than 100%? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Yes. On a multi-core system a process can run threads on several CPUs at the +same time. The maximum value is ``psutil.cpu_count() * 100``. For example, on a +4-core machine a fully-loaded process can reach 400%. The system-wide +:func:`cpu_percent` (without a :class:`Process`) always stays in the 0–100% +range because it averages across all cores. + +The returned value is explicitly *not* split evenly between all available CPUs. +This is consistent with the ``top`` UNIX utility: a busy loop on a system with +2 :term:`logical CPUs ` is reported as 100%, not 50%. Note that +Windows ``taskmgr.exe`` behaves differently (it would report 50%). To emulate +that: ``p.cpu_percent() / psutil.cpu_count()``. + +.. _faq_cpu_count: + +What is the difference between psutil, os, and multiprocessing cpu_count()? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- :func:`os.cpu_count` returns the number of :term:`logical CPUs ` + (including hyperthreads). It is the same as + ``psutil.cpu_count(logical=True)``, but psutil does not honour + :envvar:`PYTHON_CPU_COUNT` environment variable introduced in Python 3.13. +- :func:`os.process_cpu_count` (Python 3.13+) returns the number of CPUs the + calling process is **allowed to use** (respects :term:`CPU affinity` and + cgroups). The psutil equivalent is ``len(psutil.Process().cpu_affinity())``. +- :func:`multiprocessing.cpu_count` returns the same value as + :func:`os.process_cpu_count` (Python 3.13+). +- :func:`psutil.cpu_count` with ``logical=False`` returns the number of + :term:`physical cores `, which has no stdlib equivalent. + +Memory +------ + +.. _faq_virtual_memory_available: + +What is the difference between virtual_memory() available and free? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:func:`virtual_memory` returns both :field:`free` and :field:`available`, but +they measure different things: + +- :field:`free`: memory that is not being used at all. +- :field:`available`: how much memory can be given to processes without + :term:`swapping `. This includes reclaimable + :term:`caches ` and :term:`buffers` that the OS can reclaim under + pressure. + +In practice, :field:`available` is almost always the metric you want when +monitoring memory. :field:`free` can be misleadingly low on systems where the +OS aggressively uses RAM for caches (which is normal and healthy). On Windows, +:field:`free` and :field:`available` are the same value. + +.. _faq_memory_rss_vs_vms: + +What is the difference between RSS and VMS? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- :field:`rss` (:term:`Resident Set Size `) is the amount of physical + memory (RAM) currently mapped into the process. +- :field:`vms` (:term:`Virtual Memory Size `) is the total virtual address + space of the process, including memory that has been + :term:`swapped out `, shared libraries, and + :term:`memory-mapped files `. + +:field:`rss` is generally the most useful metric for answering "how much RAM is +this process using?". Note that it includes :term:`shared memory`, so it may +overestimate actual usage when compared across processes. :field:`vms` is +generally larger and can be misleadingly high, as it includes memory that is +not resident in physical RAM. Both values are portable across platforms and are +returned by :meth:`Process.memory_info`. + +.. _faq_memory_footprint: + +When should I use memory_footprint() vs memory_info()? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:meth:`Process.memory_info` returns :field:`rss` +(:term:`Resident Set Size `), which includes +:term:`shared libraries ` counted in every process that uses +them. For example, if ``libc`` uses 2 MB and 100 processes map it, each process +includes those 2 MB in its :field:`rss`. + +:meth:`Process.memory_footprint` returns :field:`uss` +(:term:`Unique Set Size `), i.e. :term:`private memory` of the process. It +represents the amount of memory that would be freed if the process were +terminated right now. It is more accurate than :term:`RSS`, but substantially +slower and requires higher privileges. On Linux it also returns :field:`pss` +(:term:`Proportional Set Size `) and :term:`swap `. + +.. _faq_used_plus_free: + +Why does virtual_memory() used + free != total? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Because some memory (like :term:`page cache` and :term:`buffers`) is +reclaimable and accounted separately: + +.. code-block:: pycon + + >>> import psutil + >>> m = psutil.virtual_memory() + >>> m.used + m.free == m.total + False + +The :field:`available` field already includes this reclaimable memory and is +the best indicator of memory pressure. See :ref:`faq_virtual_memory_available`. diff --git a/docs/funding.rst b/docs/funding.rst new file mode 100644 index 0000000000..72234552be --- /dev/null +++ b/docs/funding.rst @@ -0,0 +1,50 @@ +Funding +======= + +psutil is free and open source software, maintained by a single developer in +his spare time. It is among the +`top 100 most-downloaded Python packages `_, used by +millions of developers and hundreds of thousands of projects worldwide, +including TensorFlow, PyTorch, Home Assistant, Ansible, and Celery. + +Keeping up with bug reports, platform compatibility, user support, and ongoing +maintenance has become increasingly difficult to sustain as a one-person +effort. Financial support helps dedicate more time to the project and ensures +its long-term health. + +How to fund +----------- + +There are several ways to support psutil development financially: + +`GitHub Sponsors `_ + The preferred platform for recurring or one-time sponsorships. GitHub + matches contributions for eligible sponsors. + +`Open Collective `_ + Transparent, open funding for individuals and companies. Expenses and + income are publicly visible. + +`PayPal `_ + One-time donations via PayPal. + +For companies +------------- + +If your company relies on psutil in production, consider becoming a sponsor. +Benefits include: + +- Your logo displayed on the `psutil homepage `_ and the + `GitHub repository `_. +- Priority response to bug reports and feature requests. +- The assurance that the library you depend on is actively maintained. + +To discuss sponsorship options, contact the author at g.rodola@gmail.com. + +Current sponsors +---------------- + +.. raw:: html + :file: _sponsors.html + +Past donors are listed in the :doc:`credits` page. diff --git a/docs/glossary.rst b/docs/glossary.rst new file mode 100644 index 0000000000..e152076ae8 --- /dev/null +++ b/docs/glossary.rst @@ -0,0 +1,355 @@ +Glossary +======== + +.. glossary:: + :sorted: + + anonymous memory + + RAM used by the program that is not associated with any file (unlike the + :term:`page cache`), such as the :term:`heap`, the stack, and other + memory allocated directly by the program (e.g. via ``malloc()``). + Anonymous pages have no on-disk counterpart and must be written to + :term:`swap memory` if evicted. Exposed by psutil via the :field:`rss_anon` + field of :meth:`Process.memory_extras` (total resident anonymous pages) + and the :field:`anonymous` field of :meth:`Process.memory_maps` (per + mapping). Anonymous regions are also visible in the :field:`path` column + of :meth:`Process.memory_maps` as ``"[heap]"``, ``"[stack]"``, or an + empty string. + + available memory + + The amount of RAM that can be given to processes without the system going + into :term:`swap `. This is the right field to watch for + memory pressure, not :field:`free`. :field:`free` is often deceptively low + because the OS keeps recently freed pages as reclaimable cache; those pages + are counted in :field:`available` but not in :field:`free`. + A monitoring alert should fire on :field:`available` (or :field:`percent`) + falling below a threshold, not on :field:`free`. See :func:`virtual_memory`. + + buffers + + Kernel memory used to cache filesystem metadata such as + superblocks, inodes, and directory entries. Distinct from the + :term:`page cache`, which caches file *contents*. + Like the page cache, buffer memory is reclaimable: the OS can + free it under memory pressure. + Reported as the :field:`buffers` field of :func:`virtual_memory` + (Linux, BSD). + + busy_time + + A :term:`cumulative counter` (milliseconds) tracking the time a disk + device spent actually performing I/O, as reported in the :field:`busy_time` + field of :func:`disk_io_counters` (Linux and FreeBSD only). To use it, + sample twice and divide the delta by elapsed time to get a utilization + percentage (analogous to CPU percent but for disks). A value close to + 100% means the disk is saturated. + + .. seealso:: :ref:`Real-time disk I/O percent recipe ` + + CPU affinity + + A property of a process (or thread) that restricts which + :term:`logical CPUs ` it is allowed to run on. For example, + pinning a process to CPU 0 and CPU 1 prevents the OS scheduler from + moving it to other cores. This could be useful, e.g., for benchmarking. + See :meth:`Process.cpu_affinity`. + + context switch + + Occurs when the CPU stops executing one process or thread for another. + Frequent switching can indicate high system + load or thread contention. See :meth:`Process.num_ctx_switches` + and :func:`cpu_stats` (:field:`ctx_switches` field). + A :field:`voluntary` context switch occurs when a process gives up the + CPU, usually because it's waiting for something (I/O, a sleep, a mutex + lock). High rates are normal for I/O-bound workloads (e.g. a web server) + and usually point to I/O or locking as the bottleneck. + An :field:`involuntary` context switch occurs when the OS forcibly takes + the CPU from the process. High rates mean the process has more work to do + but is being kicked off the core. This usually indicates too many active + threads/processes competing for too few CPU cores. + + cumulative counter + + A field whose value only increases over time (since boot or process + creation) and never resets. Examples include :func:`cpu_times`, + :func:`disk_io_counters`, :func:`net_io_counters`, + :meth:`Process.io_counters`, and :meth:`Process.num_ctx_switches`. + The raw value is rarely useful on its own; divide the delta between + two samples by the elapsed time to get a meaningful rate (e.g. + bytes per second, context switches per second). + + file descriptor + + An integer handle used by UNIX processes to reference open files, + sockets, pipes, and other I/O resources. On Windows the equivalent + are :term:`handles `. Leaking file descriptors (opening without + closing) eventually causes ``EMFILE`` / ``Too many open files`` errors. + See :meth:`Process.num_fds` and :meth:`Process.open_files`. + + handle + + On Windows, an opaque reference to a kernel object such as a file, + thread, process, event or mutex. Handles are the Windows equivalent of + UNIX :term:`file descriptors `. Each open handle + consumes a small amount of kernel memory. Leaking / unclosed + handles eventually causes ``ERROR_NO_MORE_FILES`` or similar errors. See + :meth:`Process.num_handles`. + + hardware interrupt + + A signal sent by a hardware device (disk controller, :term:`NIC`, keyboard) + to the CPU to request attention. Each interrupt briefly preempts + whatever the CPU was doing. Reported as the :field:`interrupts` field of + :func:`cpu_stats` and :field:`irq` field of :func:`cpu_times`. + A very high rate may indicate a misbehaving device driver or a heavily + loaded :term:`NIC`. Also see :term:`soft interrupt`. + + heap + + The memory region managed by the platform's native C allocator + (e.g. glibc's ``malloc`` on Linux, ``jemalloc`` on FreeBSD, + ``HeapAlloc`` on Windows). When a C extension calls ``malloc()`` + and never calls ``free()``, the leaked bytes show up here but + are not always visible to Python's memory tracking tools + (:mod:`tracemalloc`, :func:`sys.getsizeof`) or :term:`RSS` / :term:`VMS`. + :func:`heap_info` exposes the current state of the heap, and + :func:`heap_trim` asks the allocator to release unused portions + of it. Together they provide a way to detect memory leaks in C + extensions that standard process-level metrics would otherwise miss. + + involuntary context switch + + See :term:`context switch`. + + iowait + + A CPU time field (Linux, SunOS, AIX) measuring time spent by the CPU + waiting for I/O operations to complete. High iowait indicates a + disk or network bottleneck. It is reported as part of + :func:`cpu_times` but is *not* included in the idle counter. + To get it as a percentage: ``psutil.cpu_times_percent(interval=1).iowait``. + Note that this is a CPU metric, not a disk metric: it measures how much + CPU time is wasted waiting, not how busy the disk is. See also + :term:`busy_time` for actual disk utilization. + + ionice + + An I/O scheduling priority that controls how much disk bandwidth a + process receives. On Linux three scheduling classes are supported: + :data:`IOPRIO_CLASS_RT` (real-time), :data:`IOPRIO_CLASS_BE` + (best-effort, the default), and :data:`IOPRIO_CLASS_IDLE`. See + :meth:`Process.ionice`. + + logical CPU + + A CPU as seen by the operating system scheduler. On systems with + *hyper-threading* each physical core exposes two logical CPUs, so a + 4-core hyper-threaded chip has 8 logical CPUs. This is the count + returned by :func:`cpu_count` (the default) and the number of + entries returned by ``cpu_percent(percpu=True)``. See also + :term:`physical CPU`. + + mapped memory + + A region of a process's virtual address space typically created via + ``mmap()``. Mappings can be file-backed (e.g. shared libraries, + memory-mapped files) or :term:`anonymous `. + Each mapping has its own permissions and memory accounting fields + (:term:`RSS`, :term:`PSS`, private / shared pages). + See :meth:`Process.memory_maps`. + + NIC + + *Network Interface Card*, a hardware or virtual network interface. + psutil uses this term when referring to per-interface network + statistics. See :func:`net_if_addrs` and :func:`net_if_stats`. + + nice + + A process priority value that influences how much CPU time the OS + scheduler gives to a process. Lower nice values mean higher priority. The + range is −20 (highest priority) to 19 (lowest) on UNIX; on Windows the + concept maps to :ref:`priority constants `. See + :meth:`Process.nice`. + + page cache + + RAM used to cache data of regular files on disk. + When a process reads a file, the data stays in the page cache, and when + it writes, the data is first stored in the cache before being written to + disk. Subsequent reads or writes can be served from RAM without disk I/O, + making access fast. The OS reclaims page cache automatically under memory + pressure, so a large cache is healthy. Shown as the :field:`cached` field + of :func:`virtual_memory` on Linux/BSD. + + page fault + + An event that occurs when a process accesses a virtual memory page that + is not currently mapped in physical RAM. A :field:`minor` fault occurs + when a page is already in physical RAM (e.g., in the :term:`page cache` + or other :term:`shared memory`), but it's not yet mapped into the + process's virtual address space, so no disk I/O is required (fast). A + :field:`major` fault requires reading the page from disk, and is + significantly more expensive. Many major faults may indicate memory + pressure or excessive swapping. See :meth:`Process.page_faults`. + + peak_rss + + The highest :term:`RSS` value a process has ever reached since it + started (memory high-water mark). Available via + :meth:`Process.memory_info` (BSD, Windows) and + :meth:`Process.memory_extras` (Linux). Useful for capacity + planning and leak detection: if :field:`peak_rss` keeps growing across + successive runs or over time, the process is likely leaking memory. + See also :term:`peak_vms`. + + peak_vms + + The highest :term:`VMS` value a process has ever reached since it + started. Available via :meth:`Process.memory_extras` (Linux) and + :meth:`Process.memory_info` (Windows). On Windows this maps to + ``PeakPagefileUsage`` (peak :term:`private ` committed + memory), which is not the same as UNIX VMS. See also :term:`peak_rss`. + + physical CPU + + An actual hardware CPU core on the motherboard, as opposed to a + :term:`logical CPU`. A single physical core may appear as multiple + logical CPUs when hyper-threading is enabled. The physical count is + returned by ``cpu_count(logical=False)``. + + private memory + + Memory pages not shared with any other process, such as the + :term:`heap`, the stack, and other allocations made directly by the + program, e.g. via ``malloc()``. + :term:`USS`, returned by :meth:`Process.memory_footprint`, measures + exactly the private memory of a process, that is the bytes that would be + freed if the process exited. At a per-mapping level, the + :field:`private_clean` and :field:`private_dirty` fields of + :meth:`Process.memory_maps` (Linux) and the :field:`private` field (FreeBSD) + break it down further. + + PSS + + *Proportional Set Size*, the amount of RAM used by a process, + where :term:`shared memory` pages are divided proportionally among all + processes that map them. PSS gives a fairer per-process memory estimate + than :term:`RSS` when shared libraries are involved. Available on Linux + via :meth:`Process.memory_footprint`. + + resource limit + + A per-process cap on a system resource enforced by the kernel (POSIX + :data:`RLIMIT_* ` constants). + Each limit has a *soft* value (the current enforcement threshold, which + the process may raise up to the hard limit) and a *hard* value + (the ceiling, settable only by root). + Common limits include :data:`RLIM_INFINITY` (open file descriptors), + :data:`RLIMIT_AS` (virtual address space), and :data:`RLIMIT_CPU` + (CPU time in seconds). See :meth:`Process.rlimit`. + + RSS + + *Resident Set Size*, the amount of physical RAM currently used by a + process. This includes :term:`shared memory` pages. It is the most + commonly reported memory metric (shown as ``RES`` in ``top``), but can be + misleading because :term:`shared memory` is counted in full for each + process that maps it. + See :meth:`Process.memory_info`. + + soft interrupt + + Deferred work scheduled by a :term:`hardware interrupt` handler to + run later in a less time-critical context (e.g. network packet + processing, block I/O completion). Using soft interrupts lets the + hardware interrupt return quickly while the heavier processing + happens shortly after. Reported as the :field:`soft_interrupts` field of + :func:`cpu_stats`. A high rate usually points to heavy network or + disk I/O throughput rather than a hardware problem. + + shared memory + + Memory pages mapped by more than one process at the same time. The most + common example is shared libraries (e.g. ``libc.so``): the OS loads them + once and lets every process that needs them map the same physical pages, + saving RAM. Shared pages are counted in full in :term:`RSS` for every + process that maps them. :term:`PSS` corrects for this by splitting each + shared page proportionally among the processes that use it. + See also :term:`private memory`. + + Exposed by psutil as the :field:`shared` field of :func:`virtual_memory` and + :meth:`Process.memory_info` (Linux), the :field:`rss_shmem` field of + :meth:`Process.memory_extras` (Linux), and the :field:`shared_clean` / + :field:`shared_dirty` fields of :meth:`Process.memory_maps` (Linux). + + swap-in + + Memory moved from disk (:term:`swap `) back into RAM. + Reported as the :field:`sin` :term:`cumulative counter` of + :func:`swap_memory`. A non-zero :field:`sin` rate usually means the system + is bringing memory back into RAM for processes to use. See also + :term:`swap-out`. + + swap-out + + Memory moved from RAM to disk (:term:`swap `). + Reported as the :field:`sout` :term:`cumulative counter` of + :func:`swap_memory`. A non-zero :field:`sout` rate indicates memory + pressure: the system is running low on RAM and must move data to disk, + which can slow performance. See also :term:`swap-in`. + + .. seealso:: + - :ref:`swap activity recipe ` + - :term:`thrashing` + + swap memory + + Disk space used as an extension of physical RAM. When the OS runs out of + RAM, it moves memory to disk to free space (:term:`swap-out`), and moves + it back into RAM (:term:`swap-in`) when a process needs it. If RAM is + full, the OS may first swap out other pages to make room. Swap prevents + out-of-memory crashes, but is much slower than RAM, so heavy swapping can + significantly degrade performance. See :func:`swap_memory` and + :ref:`swap activity recipe `. + + thrashing + + A condition where the system spends more time moving memory between RAM + and disk (:term:`swap `) than doing actual work, because memory + demand exceeds available RAM. The symptom is high and sustained rates on + both :field:`sin` and :field:`sout` from :func:`swap_memory`. + As a result, the system becomes very slow or unresponsive. CPU utilization + may look low while everything is waiting on disk I/O. + + USS + + *Unique Set Size*, the :term:`private memory` of a process, that belongs + exclusively to it, and which would be freed if it exited. It excludes + :term:`shared memory` pages entirely, making it the most accurate + single-process memory metric. Available on Linux, macOS, and Windows via + :meth:`Process.memory_footprint`. + + voluntary context switch + + See :term:`context switch`. + + VMS + + *Virtual Memory Size*, the total virtual address space reserved by a + process, including mapped files, :term:`shared memory`, stack, and + :term:`heap`, regardless of whether those pages are currently in RAM or + in :term:`swap memory`. VMS is almost always much larger than :term:`RSS` + because most virtual pages are never actually loaded into RAM. See + :meth:`Process.memory_info`. + + zombie process + + A process that has exited but whose entry remains in the process + table until its parent calls ``wait()``. Zombies hold a PID but consume + no CPU or memory. + + .. seealso:: :ref:`faq_zombie_process` diff --git a/docs/index.rst b/docs/index.rst index dbd1d329bc..3e96284fb2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,1784 +1,304 @@ .. module:: psutil :synopsis: psutil module -.. moduleauthor:: Giampaolo Rodola' - -psutil documentation -==================== - -Quick links ------------ - -* `Home page `__ -* `Install `_ -* `Blog `__ -* `Forum `__ -* `Download `__ -* `Development guide `_ -* `What's new `__ - -About ------ - -psutil (python system and process utilities) is a cross-platform library for -retrieving information on running -**processes** and **system utilization** (CPU, memory, disks, network) in -**Python**. -It is useful mainly for **system monitoring**, **profiling** and **limiting -process resources** and **management of running processes**. -It implements many functionalities offered by command line tools -such as: *ps, top, lsof, netstat, ifconfig, who, df, kill, free, nice, -ionice, iostat, iotop, uptime, pidof, tty, taskset, pmap*. -It currently supports **Linux, Windows, OSX, Sun Solaris, FreeBSD, OpenBSD** -and **NetBSD**, both **32-bit** and **64-bit** architectures, with Python -versions from **2.6 to 3.5** (users of Python 2.4 and 2.5 may use -`2.1.3 `__ version). -`PyPy `__ is also known to work. - -The psutil documentation you're reading is distributed as a single HTML page. - -System related functions -======================== - -CPU ---- - -.. function:: cpu_times(percpu=False) - - Return system CPU times as a namedtuple. - Every attribute represents the seconds the CPU has spent in the given mode. - The attributes availability varies depending on the platform: - - - **user** - - **system** - - **idle** - - Platform-specific fields: - - - **nice** *(UNIX)* - - **iowait** *(Linux)* - - **irq** *(Linux, BSD)* - - **softirq** *(Linux)* - - **steal** *(Linux 2.6.11+)* - - **guest** *(Linux 2.6.24+)* - - **guest_nice** *(Linux 3.2.0+)* - - **interrupt** *(Windows)* - - **dpc** *(Windows)* - - When *percpu* is ``True`` return a list of namedtuples for each logical CPU - on the system. - First element of the list refers to first CPU, second element to second CPU - and so on. - The order of the list is consistent across calls. - Example output on Linux: - - >>> import psutil - >>> psutil.cpu_times() - scputimes(user=17411.7, nice=77.99, system=3797.02, idle=51266.57, iowait=732.58, irq=0.01, softirq=142.43, steal=0.0, guest=0.0, guest_nice=0.0) - - .. versionchanged:: 4.1.0 added *interrupt* and *dpc* fields on Windows. - -.. function:: cpu_percent(interval=None, percpu=False) - - Return a float representing the current system-wide CPU utilization as a - percentage. When *interval* is > ``0.0`` compares system CPU times elapsed - before and after the interval (blocking). - When *interval* is ``0.0`` or ``None`` compares system CPU times elapsed - since last call or module import, returning immediately. - That means the first time this is called it will return a meaningless ``0.0`` - value which you are supposed to ignore. - In this case is recommended for accuracy that this function be called with at - least ``0.1`` seconds between calls. - When *percpu* is ``True`` returns a list of floats representing the - utilization as a percentage for each CPU. - First element of the list refers to first CPU, second element to second CPU - and so on. The order of the list is consistent across calls. - - >>> import psutil - >>> # blocking - >>> psutil.cpu_percent(interval=1) - 2.0 - >>> # non-blocking (percentage since last call) - >>> psutil.cpu_percent(interval=None) - 2.9 - >>> # blocking, per-cpu - >>> psutil.cpu_percent(interval=1, percpu=True) - [2.0, 1.0] - >>> - - .. warning:: - the first time this function is called with *interval* = ``0.0`` or ``None`` - it will return a meaningless ``0.0`` value which you are supposed to - ignore. - -.. function:: cpu_times_percent(interval=None, percpu=False) - - Same as :func:`cpu_percent()` but provides utilization percentages for each - specific CPU time as is returned by - :func:`psutil.cpu_times(percpu=True)`. - *interval* and - *percpu* arguments have the same meaning as in :func:`cpu_percent()`. - - .. warning:: - the first time this function is called with *interval* = ``0.0`` or - ``None`` it will return a meaningless ``0.0`` value which you are supposed - to ignore. - - .. versionchanged:: - 4.1.0 two new *interrupt* and *dpc* fields are returned on Windows. - -.. function:: cpu_count(logical=True) - - Return the number of logical CPUs in the system (same as - `os.cpu_count() `__ - in Python 3.4). - If *logical* is ``False`` return the number of physical cores only (hyper - thread CPUs are excluded). Return ``None`` if undetermined. - - >>> import psutil - >>> psutil.cpu_count() - 4 - >>> psutil.cpu_count(logical=False) - 2 - >>> - -.. function:: cpu_stats() - - Return various CPU statistics as a namedtuple: - - - **ctx_switches**: - number of context switches (voluntary + involuntary) since boot. - - **interrupts**: - number of interrupts since boot. - - **soft_interrupts**: - number of software interrupts since boot. Always set to ``0`` on Windows - and SunOS. - - **syscalls**: number of system calls since boot. Always set to ``0`` on - Linux. - - Example (Linux): - - .. code-block:: python - - >>> import psutil - >>> psutil.cpu_stats() - scpustats(ctx_switches=20455687, interrupts=6598984, soft_interrupts=2134212, syscalls=0) - - .. versionadded:: 4.1.0 - - -Memory ------- - -.. function:: virtual_memory() - - Return statistics about system memory usage as a namedtuple including the - following fields, expressed in bytes: - - - **total**: total physical memory available. - - **available**: the actual amount of available memory that can be given - instantly to processes that request more memory in bytes; this is - calculated by summing different memory values depending on the platform - (e.g. free + buffers + cached on Linux) and it is supposed to be used to - monitor actual memory usage in a cross platform fashion. - - **percent**: the percentage usage calculated as - ``(total - available) / total * 100``. - - **used**: memory used, calculated differently depending on the platform and - designed for informational purposes only. - - **free**: memory not being used at all (zeroed) that is readily available; - note that this doesn't reflect the actual memory available (use 'available' - instead). - - Platform-specific fields: - - - **active** *(UNIX)*: memory currently in use or very recently used, and so - it is in RAM. - - **inactive** *(UNIX)*: memory that is marked as not used. - - **buffers** *(Linux, BSD)*: cache for things like file system metadata. - - **cached** *(Linux, BSD)*: cache for various things. - - **shared** *(Linux, BSD)*: memory that may be simultaneously accessed by - multiple processes. - - **wired** *(BSD, OSX)*: memory that is marked to always stay in RAM. It is - never moved to disk. - - The sum of **used** and **available** does not necessarily equal **total**. - On Windows **available** and **free** are the same. - See `scripts/meminfo.py `__ - script providing an example on how to convert bytes in a human readable form. - - .. note:: if you just want to know how much physical memory is left in a - cross platform fashion simply rely on the **available** field. - - >>> import psutil - >>> mem = psutil.virtual_memory() - >>> mem - svmem(total=10367352832, available=6472179712, percent=37.6, used=8186245120, free=2181107712, active=4748992512, inactive=2758115328, buffers=790724608, cached=3500347392, shared=787554304) - >>> - >>> THRESHOLD = 100 * 1024 * 1024 # 100MB - >>> if mem.available <= THRESHOLD: - ... print("warning") - ... - >>> - - .. versionchanged:: 4.2.0 added *shared* metrics on Linux. - -.. function:: swap_memory() - - Return system swap memory statistics as a namedtuple including the following - fields: - - * **total**: total swap memory in bytes - * **used**: used swap memory in bytes - * **free**: free swap memory in bytes - * **percent**: the percentage usage calculated as ``(total - available) / total * 100`` - * **sin**: the number of bytes the system has swapped in from disk - (cumulative) - * **sout**: the number of bytes the system has swapped out from disk - (cumulative) - - **sin** and **sout** on Windows are always set to ``0``. - See `scripts/meminfo.py `__ - script providing an example on how to convert bytes in a human readable form. - - >>> import psutil - >>> psutil.swap_memory() - sswap(total=2097147904L, used=886620160L, free=1210527744L, percent=42.3, sin=1050411008, sout=1906720768) - -Disks ------ - -.. function:: disk_partitions(all=False) - - Return all mounted disk partitions as a list of namedtuples including device, - mount point and filesystem type, similarly to "df" command on UNIX. If *all* - parameter is ``False`` return physical devices only (e.g. hard disks, cd-rom - drives, USB keys) and ignore all others (e.g. memory partitions such as - `/dev/shm `__). - Namedtuple's **fstype** field is a string which varies depending on the - platform. - On Linux it can be one of the values found in /proc/filesystems (e.g. - ``'ext3'`` for an ext3 hard drive o ``'iso9660'`` for the CD-ROM drive). - On Windows it is determined via - `GetDriveType `__ - and can be either ``"removable"``, ``"fixed"``, ``"remote"``, ``"cdrom"``, - ``"unmounted"`` or ``"ramdisk"``. On OSX and BSD it is retrieved via - `getfsstat(2) `__. See - `disk_usage.py `__ - script providing an example usage. - - >>> import psutil - >>> psutil.disk_partitions() - [sdiskpart(device='/dev/sda3', mountpoint='/', fstype='ext4', opts='rw,errors=remount-ro'), - sdiskpart(device='/dev/sda7', mountpoint='/home', fstype='ext4', opts='rw')] - -.. function:: disk_usage(path) - - Return disk usage statistics about the given *path* as a namedtuple including - **total**, **used** and **free** space expressed in bytes, plus the - **percentage** usage. - `OSError `__ is - raised if *path* does not exist. - Starting from `Python 3.3 `__ this is - also available as - `shutil.disk_usage() `__. - See `disk_usage.py `__ script providing an example usage. - - >>> import psutil - >>> psutil.disk_usage('/') - sdiskusage(total=21378641920, used=4809781248, free=15482871808, percent=22.5) - - .. note:: - UNIX usually reserves 5% of the total disk space for the root user. - *total* and *used* fields on UNIX refer to the overall total and used - space, whereas *free* represents the space available for the **user** and - *percent* represents the **user** utilization (see - `source code `__). - That is why *percent* value may look 5% bigger than what you would expect - it to be. - Also note that both 4 values match "df" cmdline utility. - - .. versionchanged:: - 4.3.0 *percent* value takes root reserved space into account. - -.. function:: disk_io_counters(perdisk=False) - - Return system-wide disk I/O statistics as a namedtuple including the - following fields: - - - **read_count**: number of reads - - **write_count**: number of writes - - **read_bytes**: number of bytes read - - **write_bytes**: number of bytes written - - Platform-specific fields: - - - **read_time**: (all except *NetBSD* and *OpenBSD*) time spent reading from - disk (in milliseconds) - - **write_time**: (all except *NetBSD* and *OpenBSD*) time spent writing to disk - (in milliseconds) - - **busy_time**: (*Linux*, *FreeBSD*) time spent doing actual I/Os (in - milliseconds) - - **read_merged_count** (*Linux*): number of merged reads - (see `iostat doc `__) - - **write_merged_count** (*Linux*): number of merged writes - (see `iostats doc `__) - - If *perdisk* is ``True`` return the same information for every physical disk - installed on the system as a dictionary with partition names as the keys and - the namedtuple described above as the values. - See `scripts/iotop.py `__ - for an example application. - - >>> import psutil - >>> psutil.disk_io_counters() - sdiskio(read_count=8141, write_count=2431, read_bytes=290203, write_bytes=537676, read_time=5868, write_time=94922) - >>> - >>> psutil.disk_io_counters(perdisk=True) - {'sda1': sdiskio(read_count=920, write_count=1, read_bytes=2933248, write_bytes=512, read_time=6016, write_time=4), - 'sda2': sdiskio(read_count=18707, write_count=8830, read_bytes=6060, write_bytes=3443, read_time=24585, write_time=1572), - 'sdb1': sdiskio(read_count=161, write_count=0, read_bytes=786432, write_bytes=0, read_time=44, write_time=0)} - - .. warning:: - on some systems such as Linux, on a very busy or long-lived system these - numbers may wrap (restart from zero), see - `issues #802 `__. - Applications should be prepared to deal with that. - - .. versionchanged:: - 4.0.0 added *busy_time* (Linux, FreeBSD), *read_merged_count* and - *write_merged_count* (Linux) fields. - - .. versionchanged:: - 4.0.0 NetBSD no longer has *read_time* and *write_time* fields. - -Network -------- - -.. function:: net_io_counters(pernic=False) - - Return system-wide network I/O statistics as a namedtuple including the - following attributes: - - - **bytes_sent**: number of bytes sent - - **bytes_recv**: number of bytes received - - **packets_sent**: number of packets sent - - **packets_recv**: number of packets received - - **errin**: total number of errors while receiving - - **errout**: total number of errors while sending - - **dropin**: total number of incoming packets which were dropped - - **dropout**: total number of outgoing packets which were dropped (always 0 - on OSX and BSD) - - If *pernic* is ``True`` return the same information for every network - interface installed on the system as a dictionary with network interface - names as the keys and the namedtuple described above as the values. - See `scripts/nettop.py `__ - for an example application. - - >>> import psutil - >>> psutil.net_io_counters() - snetio(bytes_sent=14508483, bytes_recv=62749361, packets_sent=84311, packets_recv=94888, errin=0, errout=0, dropin=0, dropout=0) - >>> - >>> psutil.net_io_counters(pernic=True) - {'lo': snetio(bytes_sent=547971, bytes_recv=547971, packets_sent=5075, packets_recv=5075, errin=0, errout=0, dropin=0, dropout=0), - 'wlan0': snetio(bytes_sent=13921765, bytes_recv=62162574, packets_sent=79097, packets_recv=89648, errin=0, errout=0, dropin=0, dropout=0)} - - .. warning:: - on some systems such as Linux, on a very busy or long-lived system these - numbers may wrap (restart from zero), see - `issues #802 `__. - Applications should be prepared to deal with that. - -.. function:: net_connections(kind='inet') - - Return system-wide socket connections as a list of namedtuples. - Every namedtuple provides 7 attributes: - - - **fd**: the socket file descriptor, if retrievable, else ``-1``. - If the connection refers to the current process this may be passed to - `socket.fromfd() `__ - to obtain a usable socket object. - - **family**: the address family, either `AF_INET - `__, - `AF_INET6 `__ - or `AF_UNIX `__. - - **type**: the address type, either `SOCK_STREAM - `__ or - `SOCK_DGRAM - `__. - - **laddr**: the local address as a ``(ip, port)`` tuple or a ``path`` - in case of AF_UNIX sockets. - - **raddr**: the remote address as a ``(ip, port)`` tuple or an absolute - ``path`` in case of UNIX sockets. - When the remote endpoint is not connected you'll get an empty tuple - (AF_INET*) or ``None`` (AF_UNIX). - On Linux AF_UNIX sockets will always have this set to ``None``. - - **status**: represents the status of a TCP connection. The return value - is one of the :data:`psutil.CONN_* ` constants - (a string). - For UDP and UNIX sockets this is always going to be - :const:`psutil.CONN_NONE`. - - **pid**: the PID of the process which opened the socket, if retrievable, - else ``None``. On some platforms (e.g. Linux) the availability of this - field changes depending on process privileges (root is needed). - - The *kind* parameter is a string which filters for connections that fit the - following criteria: - - .. table:: - - +----------------+-----------------------------------------------------+ - | **Kind value** | **Connections using** | - +================+=====================================================+ - | "inet" | IPv4 and IPv6 | - +----------------+-----------------------------------------------------+ - | "inet4" | IPv4 | - +----------------+-----------------------------------------------------+ - | "inet6" | IPv6 | - +----------------+-----------------------------------------------------+ - | "tcp" | TCP | - +----------------+-----------------------------------------------------+ - | "tcp4" | TCP over IPv4 | - +----------------+-----------------------------------------------------+ - | "tcp6" | TCP over IPv6 | - +----------------+-----------------------------------------------------+ - | "udp" | UDP | - +----------------+-----------------------------------------------------+ - | "udp4" | UDP over IPv4 | - +----------------+-----------------------------------------------------+ - | "udp6" | UDP over IPv6 | - +----------------+-----------------------------------------------------+ - | "unix" | UNIX socket (both UDP and TCP protocols) | - +----------------+-----------------------------------------------------+ - | "all" | the sum of all the possible families and protocols | - +----------------+-----------------------------------------------------+ - - On OSX this function requires root privileges. - To get per-process connections use :meth:`Process.connections`. - Also, see - `netstat.py sample script `__. - Example: - - >>> import psutil - >>> psutil.net_connections() - [pconn(fd=115, family=, type=, laddr=('10.0.0.1', 48776), raddr=('93.186.135.91', 80), status='ESTABLISHED', pid=1254), - pconn(fd=117, family=, type=, laddr=('10.0.0.1', 43761), raddr=('72.14.234.100', 80), status='CLOSING', pid=2987), - pconn(fd=-1, family=, type=, laddr=('10.0.0.1', 60759), raddr=('72.14.234.104', 80), status='ESTABLISHED', pid=None), - pconn(fd=-1, family=, type=, laddr=('10.0.0.1', 51314), raddr=('72.14.234.83', 443), status='SYN_SENT', pid=None) - ...] - - .. note:: - (OSX) :class:`psutil.AccessDenied` is always raised unless running as root - (lsof does the same). - - .. note:: - (Solaris) UNIX sockets are not supported. - - .. versionadded:: 2.1.0 - -.. function:: net_if_addrs() - - Return the addresses associated to each NIC (network interface card) - installed on the system as a dictionary whose keys are the NIC names and - value is a list of namedtuples for each address assigned to the NIC. - Each namedtuple includes 5 fields: - - - **family** - - **address** - - **netmask** - - **broadcast** - - **ptp** - - *family* can be either - `AF_INET `__, - `AF_INET6 `__ - or :const:`psutil.AF_LINK`, which refers to a MAC address. - *address* is the primary address and it is always set. - *netmask*, *broadcast* and *ptp* may be ``None``. - *ptp* stands for "point to point" and references the destination address on a - point to point interface (typically a VPN). - *broadcast* and *ptp* are mutually exclusive. - *netmask*, *broadcast* and *ptp* are not supported on Windows and are set to - ``None``. - - Example:: - - >>> import psutil - >>> psutil.net_if_addrs() - {'lo': [snic(family=, address='127.0.0.1', netmask='255.0.0.0', broadcast='127.0.0.1', ptp=None), - snic(family=, address='::1', netmask='ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', broadcast=None, ptp=None), - snic(family=, address='00:00:00:00:00:00', netmask=None, broadcast='00:00:00:00:00:00', ptp=None)], - 'wlan0': [snic(family=, address='192.168.1.3', netmask='255.255.255.0', broadcast='192.168.1.255', ptp=None), - snic(family=, address='fe80::c685:8ff:fe45:641%wlan0', netmask='ffff:ffff:ffff:ffff::', broadcast=None, ptp=None), - snic(family=, address='c4:85:08:45:06:41', netmask=None, broadcast='ff:ff:ff:ff:ff:ff', ptp=None)]} - >>> - - See also `scripts/ifconfig.py `__ - for an example application. - - .. note:: - if you're interested in others families (e.g. AF_BLUETOOTH) you can use - the more powerful `netifaces `__ - extension. - - .. note:: - you can have more than one address of the same family associated with each - interface (that's why dict values are lists). - - .. note:: - *netmask*, *broadcast* and *ptp* are not supported on Windows and are set - to ``None``. - - .. versionadded:: 3.0.0 - - .. versionchanged:: 3.2.0 *ptp* field was added. - -.. function:: net_if_stats() - - Return information about each NIC (network interface card) installed on the - system as a dictionary whose keys are the NIC names and value is a namedtuple - with the following fields: - - - **isup**: a bool indicating whether the NIC is up and running. - - **duplex**: the duplex communication type; - it can be either :const:`NIC_DUPLEX_FULL`, :const:`NIC_DUPLEX_HALF` or - :const:`NIC_DUPLEX_UNKNOWN`. - - **speed**: the NIC speed expressed in mega bits (MB), if it can't be - determined (e.g. 'localhost') it will be set to ``0``. - - **mtu**: NIC's maximum transmission unit expressed in bytes. - - See also `scripts/ifconfig.py `__ - for an example application. - Example: - - >>> import psutil - >>> psutil.net_if_stats() - {'eth0': snicstats(isup=True, duplex=, speed=100, mtu=1500), - 'lo': snicstats(isup=True, duplex=, speed=0, mtu=65536)} +.. moduleauthor:: Giampaolo Rodola +.. title:: Home + +.. ============================================================================ +.. Hero +.. ============================================================================ + +.. raw:: html + + + +
    +

    psutil

    +
    Process and System Utilities for Python
    +
    + +.. container:: home-intro + + Psutil is a cross-platform library for retrieving information about running + processes and system utilization in Python. It is useful mainly for system + monitoring, profiling, limiting process resources, and managing running + processes. Psutil implements many functionalities offered by UNIX command + line tool such as *ps, top, free, iotop, netstat, ifconfig, lsof* and + others. + +.. ============================================================================ +.. Install one-liner +.. ============================================================================ + +.. raw:: html + +
    + + pip install psutil + + +
    + +.. ============================================================================ +.. Platform pills +.. ============================================================================ + +.. raw:: html + + + +.. ============================================================================ +.. Feature cards +.. ============================================================================ + +.. raw:: html + + + + +.. ============================================================================ +.. Quickstart code preview (tabs by category) +.. ============================================================================ + +.. raw:: html + + + +.. container:: home-quickstart + + .. tab-set:: + + .. tab-item:: CPU + + .. code-block:: pycon + + >>> import psutil + >>> psutil.cpu_times() + scputimes(user=3961.46, nice=169.72, system=2150.65, idle=16900.54, iowait=629.59, ...) + >>> psutil.cpu_percent(interval=1) + 4.0 + >>> psutil.cpu_count(logical=False) + 2 + >>> psutil.cpu_freq() + scpufreq(current=931.42, min=800.0, max=3500.0) + + .. container:: home-tab-more + + :ref:`See more → ` + + .. tab-item:: Memory + + .. code-block:: pycon + + >>> import psutil + >>> psutil.virtual_memory() + svmem(total=10367352832, available=6472179712, percent=37.6, used=8186245120, free=2181107712, ...) + >>> psutil.swap_memory() + sswap(total=2097147904, used=296128512, free=1801019392, percent=14.1, sin=304193536, sout=677842944) + + .. container:: home-tab-more + + :ref:`See more → ` + + .. tab-item:: Disks + + .. code-block:: pycon + + >>> import psutil + >>> psutil.disk_partitions() + [sdiskpart(device='/dev/sda1', mountpoint='/', fstype='ext4', opts='rw,nosuid'), + sdiskpart(device='/dev/sda2', mountpoint='/home', fstype='ext4', opts='rw')] + >>> psutil.disk_usage('/') + sdiskusage(total=21378641920, used=4809781248, free=15482871808, percent=22.5) + >>> psutil.disk_io_counters() + sdiskio(read_count=719566, write_count=1082197, read_bytes=18626220, write_bytes=24081764, ...) + + .. container:: home-tab-more + + :ref:`See more → ` + + .. tab-item:: Network + + .. code-block:: pycon + + >>> import psutil + >>> psutil.net_io_counters(pernic=True) + {'eth0': netio(bytes_sent=485291293, bytes_recv=6004858642, ...), + 'lo': netio(bytes_sent=2838627, bytes_recv=2838627, ...)} + >>> psutil.net_connections(kind='tcp') + [sconn(family=2, type=1, laddr=addr('10.0.0.1', 48776), raddr=addr('93.186.135.91', 80), status='ESTABLISHED', pid=1254), ...] + >>> psutil.net_if_addrs()['wlan0'] + [snicaddr(family=2, address='192.168.1.3', netmask='255.255.255.0', ...), ...] + >>> psutil.net_if_stats()['wlan0'] + snicstats(isup=True, duplex=2, speed=100, mtu=1500, flags='up,broadcast,running') + + .. container:: home-tab-more + + :ref:`See more → ` + + .. tab-item:: Sensors + + .. code-block:: pycon - .. versionadded:: 3.0.0 + >>> import psutil + >>> psutil.sensors_temperatures() + {'coretemp': [shwtemp(label='Physical id 0', current=52.0, high=100.0, critical=100.0), + shwtemp(label='Core 0', current=45.0, high=100.0, critical=100.0)], + 'acpitz': [shwtemp(label='', current=47.0, high=103.0, critical=103.0)]} + >>> psutil.sensors_fans() + {'asus': [sfan(label='cpu_fan', current=3200)]} + >>> psutil.sensors_battery() + sbattery(percent=93, secsleft=16628, power_plugged=False) + .. container:: home-tab-more -Other system info ------------------ - -.. function:: boot_time() + :ref:`See more → ` - Return the system boot time expressed in seconds since the epoch. - Example: + .. tab-item:: Processes - .. code-block:: python + .. code-block:: pycon - >>> import psutil, datetime - >>> psutil.boot_time() - 1389563460.0 - >>> datetime.datetime.fromtimestamp(psutil.boot_time()).strftime("%Y-%m-%d %H:%M:%S") - '2014-01-12 22:51:00' + >>> import psutil + >>> p = psutil.Process(7055) + >>> p.name(), p.exe(), p.cmdline() + ('python3', '/usr/bin/python3', ['/usr/bin/python3', 'main.py']) + >>> p.status() + + >>> p.cpu_percent(interval=1.0) + 12.1 + >>> p.memory_info() + pmem(rss=3164160, vms=4410163, shared=897433, text=302694, data=2422374) + >>> p.parent(), p.children(recursive=True) + (psutil.Process(pid=4699, name='bash'), [psutil.Process(pid=29835, name='python3'), ...]) + + .. container:: home-tab-more + + :ref:`See more → ` + +.. ============================================================================ +.. Adoption stats. Numbers kept in sync with adoption.rst and README.rst +.. by scripts/internal/docs/refresh_adoption_stats.py. +.. ============================================================================ -.. function:: users() +.. raw:: html - Return users currently connected on the system as a list of namedtuples - including the following fields: - - - **user**: the name of the user. - - **terminal**: the tty or pseudo-tty associated with the user, if any, - else ``None``. - - **host**: the host name associated with the entry, if any. - - **started**: the creation time as a floating point number expressed in - seconds since the epoch. - - Example:: - - >>> import psutil - >>> psutil.users() - [suser(name='giampaolo', terminal='pts/2', host='localhost', started=1340737536.0), - suser(name='giampaolo', terminal='pts/3', host='localhost', started=1340737792.0)] - -Processes -========= - -Functions ---------- - -.. function:: pids() - - Return a list of current running PIDs. To iterate over all processes - :func:`process_iter()` should be preferred. - -.. function:: pid_exists(pid) - - Check whether the given PID exists in the current process list. This is - faster than doing ``"pid in psutil.pids()"`` and should be preferred. - -.. function:: process_iter() - - Return an iterator yielding a :class:`Process` class instance for all running - processes on the local machine. - Every instance is only created once and then cached into an internal table - which is updated every time an element is yielded. - Cached :class:`Process` instances are checked for identity so that you're - safe in case a PID has been reused by another process, in which case the - cached instance is updated. - This is should be preferred over :func:`psutil.pids()` for iterating over - processes. - Sorting order in which processes are returned is - based on their PID. Example usage:: - - import psutil - - for proc in psutil.process_iter(): - try: - pinfo = proc.as_dict(attrs=['pid', 'name']) - except psutil.NoSuchProcess: - pass - else: - print(pinfo) - -.. function:: wait_procs(procs, timeout=None, callback=None) - - Convenience function which waits for a list of :class:`Process` instances to - terminate. Return a ``(gone, alive)`` tuple indicating which processes are - gone and which ones are still alive. The *gone* ones will have a new - *returncode* attribute indicating process exit status (it may be ``None``). - ``callback`` is a function which gets called every time a process terminates - (a :class:`Process` instance is passed as callback argument). Function will - return as soon as all processes terminate or when timeout occurs. Typical use - case is: - - - send SIGTERM to a list of processes - - give them some time to terminate - - send SIGKILL to those ones which are still alive - - Example:: - - import psutil - - def on_terminate(proc): - print("process {} terminated with exit code {}".format(proc, proc.returncode)) - - procs = [...] # a list of Process instances - for p in procs: - p.terminate() - gone, alive = psutil.wait_procs(procs, timeout=3, callback=on_terminate) - for p in alive: - p.kill() - -Exceptions ----------- - -.. class:: Error() - - Base exception class. All other exceptions inherit from this one. - -.. class:: NoSuchProcess(pid, name=None, msg=None) - - Raised by :class:`Process` class methods when no process with the given - pid* is found in the current process list or when a process no longer - exists. "name" is the name the process had before disappearing - and gets set only if :meth:`Process.name()` was previously called. - -.. class:: ZombieProcess(pid, name=None, ppid=None, msg=None) - - This may be raised by :class:`Process` class methods when querying a zombie - process on UNIX (Windows doesn't have zombie processes). Depending on the - method called the OS may be able to succeed in retrieving the process - information or not. - Note: this is a subclass of :class:`NoSuchProcess` so if you're not - interested in retrieving zombies (e.g. when using :func:`process_iter()`) - you can ignore this exception and just catch :class:`NoSuchProcess`. - - .. versionadded:: 3.0.0 - -.. class:: AccessDenied(pid=None, name=None, msg=None) - - Raised by :class:`Process` class methods when permission to perform an - action is denied. "name" is the name of the process (may be ``None``). - -.. class:: TimeoutExpired(seconds, pid=None, name=None, msg=None) - - Raised by :meth:`Process.wait` if timeout expires and process is still - alive. - -Process class -------------- - -.. class:: Process(pid=None) - - Represents an OS process with the given *pid*. If *pid* is omitted current - process *pid* (`os.getpid() `__) - is used. - Raise :class:`NoSuchProcess` if *pid* does not exist. - When accessing methods of this class always be prepared to catch - :class:`NoSuchProcess`, :class:`ZombieProcess` and :class:`AccessDenied` - exceptions. - `hash() `__ builtin can - be used against instances of this class in order to identify a process - univocally over time (the hash is determined by mixing process PID - and creation time). As such it can also be used with - `set()s `__. - - .. warning:: - - the way this class is bound to a process is via its **PID**. - That means that if the :class:`Process` instance is old enough and - the PID has been reused in the meantime you might end up interacting - with another process. - The only exceptions for which process identity is preemptively checked - (via PID + creation time) and guaranteed are for - :meth:`nice` (set), - :meth:`ionice` (set), - :meth:`cpu_affinity` (set), - :meth:`rlimit` (set), - :meth:`children`, - :meth:`parent`, - :meth:`suspend` - :meth:`resume`, - :meth:`send_signal`, - :meth:`terminate`, and - :meth:`kill` - methods. - To prevent this problem for all other methods you can use - :meth:`is_running()` before querying the process or use - :func:`process_iter()` in case you're iterating over all processes. - - .. attribute:: pid - - The process PID. - - .. method:: ppid() - - The process parent pid. On Windows the return value is cached after first - call. - - .. method:: name() - - The process name. - - .. method:: exe() - - The process executable as an absolute path. - On some systems this may also be an empty string. - The return value is cached after first call. - - .. method:: cmdline() - - The command line this process has been called with. - - .. method:: environ() - - The environment variables of the process as a dict. Note: this might not - reflect changes made after the process started. - - Availability: Linux, OSX, Windows - - .. versionadded:: 4.0.0 - - .. method:: create_time() - - The process creation time as a floating point number expressed in seconds - since the epoch, in - `UTC `__. - The return value is cached after first call. - - >>> import psutil, datetime - >>> p = psutil.Process() - >>> p.create_time() - 1307289803.47 - >>> datetime.datetime.fromtimestamp(p.create_time()).strftime("%Y-%m-%d %H:%M:%S") - '2011-03-05 18:03:52' - - .. method:: as_dict(attrs=None, ad_value=None) - - Utility method retrieving multiple process information as a dictionary. - If *attrs* is specified it must be a list of strings reflecting available - :class:`Process` class's attribute names (e.g. ``['cpu_times', 'name']``), - else all public (read only) attributes are assumed. *ad_value* is the - value which gets assigned to a dict key in case :class:`AccessDenied` - or :class:`ZombieProcess` exception is raised when retrieving that - particular process information. - - >>> import psutil - >>> p = psutil.Process() - >>> p.as_dict(attrs=['pid', 'name', 'username']) - {'username': 'giampaolo', 'pid': 12366, 'name': 'python'} - - .. versionchanged:: - 3.0.0 *ad_value* is used also when incurring into - :class:`ZombieProcess` exception, not only :class:`AccessDenied` - - .. method:: parent() - - Utility method which returns the parent process as a :class:`Process` - object preemptively checking whether PID has been reused. If no parent - PID is known return ``None``. - - .. method:: status() - - The current process status as a string. The returned string is one of the - :data:`psutil.STATUS_*` constants. - - .. method:: cwd() - - The process current working directory as an absolute path. - - .. method:: username() - - The name of the user that owns the process. On UNIX this is calculated by - using real process uid. - - .. method:: uids() - - The real, effective and saved user ids of this process as a - namedtuple. This is the same as - `os.getresuid() `__ - but can be used for any process PID. - - Availability: UNIX - - .. method:: gids() - - The real, effective and saved group ids of this process as a - namedtuple. This is the same as - `os.getresgid() `__ - but can be used for any process PID. - - Availability: UNIX - - .. method:: terminal() - - The terminal associated with this process, if any, else ``None``. This is - similar to "tty" command but can be used for any process PID. - - Availability: UNIX - - .. method:: nice(value=None) - - Get or set process - `niceness `__ (priority). - On UNIX this is a number which usually goes from ``-20`` to ``20``. - The higher the nice value, the lower the priority of the process. - - >>> import psutil - >>> p = psutil.Process() - >>> p.nice(10) # set - >>> p.nice() # get - 10 - >>> - - Starting from `Python 3.3 `__ this - functionality is also available as - `os.getpriority() `__ - and - `os.setpriority() `__ - (UNIX only). - On Windows this is implemented via - `GetPriorityClass `__ - and `SetPriorityClass `__ - Windows APIs and *value* is one of the - :data:`psutil.*_PRIORITY_CLASS ` - constants reflecting the MSDN documentation. - Example which increases process priority on Windows: - - >>> p.nice(psutil.HIGH_PRIORITY_CLASS) - - .. method:: ionice(ioclass=None, value=None) - - Get or set - `process I/O niceness `__ (priority). - On Linux *ioclass* is one of the - :data:`psutil.IOPRIO_CLASS_*` constants. - *value* is a number which goes from ``0`` to ``7``. The higher the value, - the lower the I/O priority of the process. On Windows only *ioclass* is - used and it can be set to ``2`` (normal), ``1`` (low) or ``0`` (very low). - The example below sets IDLE priority class for the current process, - meaning it will only get I/O time when no other process needs the disk: - - >>> import psutil - >>> p = psutil.Process() - >>> p.ionice(psutil.IOPRIO_CLASS_IDLE) # set - >>> p.ionice() # get - pionice(ioclass=, value=0) - >>> - - On Windows only *ioclass* is used and it can be set to ``2`` (normal), - ``1`` (low) or ``0`` (very low). - - Availability: Linux and Windows > Vista - - .. versionchanged:: - 3.0.0 on Python >= 3.4 the returned ``ioclass`` constant is an - `enum `__ - instead of a plain integer. - - .. method:: rlimit(resource, limits=None) - - Get or set process resource limits (see - `man prlimit `__). *resource* is one - of the :data:`psutil.RLIMIT_* ` constants. - *limits* is a ``(soft, hard)`` tuple. - This is the same as `resource.getrlimit() `__ - and `resource.setrlimit() `__ - but can be used for any process PID, not only - `os.getpid() `__. - Example: - - >>> import psutil - >>> p = psutil.Process() - >>> # process may open no more than 128 file descriptors - >>> p.rlimit(psutil.RLIMIT_NOFILE, (128, 128)) - >>> # process may create files no bigger than 1024 bytes - >>> p.rlimit(psutil.RLIMIT_FSIZE, (1024, 1024)) - >>> # get - >>> p.rlimit(psutil.RLIMIT_FSIZE) - (1024, 1024) - >>> - - Availability: Linux - - .. method:: io_counters() - - Return process I/O statistics as a namedtuple including the number of read - and write operations performed by the process and the amount of bytes read - and written. For Linux refer to - `/proc filesysem documentation `__. - On BSD there's apparently no way to retrieve bytes counters, hence ``-1`` - is returned for **read_bytes** and **write_bytes** fields. OSX is not - supported. - - >>> import psutil - >>> p = psutil.Process() - >>> p.io_counters() - pio(read_count=454556, write_count=3456, read_bytes=110592, write_bytes=0) - - Availability: all platforms except OSX and Solaris - - .. method:: num_ctx_switches() - - The number voluntary and involuntary context switches performed by - this process. - - .. method:: num_fds() - - The number of file descriptors used by this process. - - Availability: UNIX - - .. method:: num_handles() - - The number of handles used by this process. - - Availability: Windows - - .. method:: num_threads() - - The number of threads used by this process. - - .. method:: threads() - - Return threads opened by process as a list of namedtuples including thread - id and thread CPU times (user/system). On OpenBSD this method requires - root access. - - .. method:: cpu_times() - - Return a `(user, system, children_user, children_system)` namedtuple - representing the accumulated process time, in seconds (see - `explanation `__). - On Windows and OSX only *user* and *system* are filled, the others are - set to ``0``. - This is similar to - `os.times() `__ - but can be used for any process PID. - - .. versionchanged:: - 4.1.0 return two extra fields: *children_user* and *children_system*. - - .. method:: cpu_percent(interval=None) - - Return a float representing the process CPU utilization as a percentage. - The returned value refers to the utilization of a single CPU, i.e. it is - not evenly split between the number of available CPU cores. - When *interval* is > ``0.0`` compares process times to system CPU times - elapsed before and after the interval (blocking). When interval is ``0.0`` - or ``None`` compares process times to system CPU times elapsed since last - call, returning immediately. That means the first time this is called it - will return a meaningless ``0.0`` value which you are supposed to ignore. - In this case is recommended for accuracy that this function be called a - second time with at least ``0.1`` seconds between calls. - Example: - - >>> import psutil - >>> p = psutil.Process() - >>> - >>> # blocking - >>> p.cpu_percent(interval=1) - 2.0 - >>> # non-blocking (percentage since last call) - >>> p.cpu_percent(interval=None) - 2.9 - >>> - - .. note:: - a percentage > 100 is legitimate as it can result from a process with - multiple threads running on different CPU cores. - - .. note:: - the returned value is explcitly **not** split evenly between all CPUs - cores (differently from :func:`psutil.cpu_percent()`). - This means that a busy loop process running on a system with 2 CPU - cores will be reported as having 100% CPU utilization instead of 50%. - This was done in order to be consistent with UNIX's "top" utility - and also to make it easier to identify processes hogging CPU resources - (independently from the number of CPU cores). - It must be noted that in the example above taskmgr.exe on Windows will - report 50% usage instead. - To emulate Windows's taskmgr.exe behavior you can do: - ``p.cpu_percent() / psutil.cpu_count()``. - - .. warning:: - the first time this method is called with interval = ``0.0`` or - ``None`` it will return a meaningless ``0.0`` value which you are - supposed to ignore. - - .. method:: cpu_affinity(cpus=None) - - Get or set process current - `CPU affinity `__. - CPU affinity consists in telling the OS to run a certain process on a - limited set of CPUs only. The number of eligible CPUs can be obtained with - ``list(range(psutil.cpu_count()))``. ``ValueError`` will be raise on set - in case an invalid CPU number is specified. - - >>> import psutil - >>> psutil.cpu_count() - 4 - >>> p = psutil.Process() - >>> p.cpu_affinity() # get - [0, 1, 2, 3] - >>> p.cpu_affinity([0]) # set; from now on, process will run on CPU #0 only - >>> p.cpu_affinity() - [0] - >>> - >>> # reset affinity against all CPUs - >>> all_cpus = list(range(psutil.cpu_count())) - >>> p.cpu_affinity(all_cpus) - >>> - - Availability: Linux, Windows, FreeBSD - - .. versionchanged:: 2.2.0 added support for FreeBSD - - .. method:: memory_info() - - Return a namedtuple with variable fields depending on the platform - representing memory information about the process. - The "portable" fields available on all plaforms are `rss` and `vms`. - All numbers are expressed in bytes. - - +---------+---------+-------+---------+------------------------------+ - | Linux | OSX | BSD | Solaris | Windows | - +=========+=========+=======+=========+==============================+ - | rss | rss | rss | rss | rss (alias for ``wset``) | - +---------+---------+-------+---------+------------------------------+ - | vms | vms | vms | vms | vms (alias for ``pagefile``) | - +---------+---------+-------+---------+------------------------------+ - | shared | pfaults | text | | num_page_faults | - +---------+---------+-------+---------+------------------------------+ - | text | pageins | data | | peak_wset | - +---------+---------+-------+---------+------------------------------+ - | lib | | stack | | wset | - +---------+---------+-------+---------+------------------------------+ - | data | | | | peak_paged_pool | - +---------+---------+-------+---------+------------------------------+ - | dirty | | | | paged_pool | - +---------+---------+-------+---------+------------------------------+ - | | | | | peak_nonpaged_pool | - +---------+---------+-------+---------+------------------------------+ - | | | | | nonpaged_pool | - +---------+---------+-------+---------+------------------------------+ - | | | | | pagefile | - +---------+---------+-------+---------+------------------------------+ - | | | | | peak_pagefile | - +---------+---------+-------+---------+------------------------------+ - | | | | | private | - +---------+---------+-------+---------+------------------------------+ - - - **rss**: aka "Resident Set Size", this is the non-swapped physical - memory a process has used. - On UNIX it matches "top"'s RES column - (see `doc `__). - On Windows this is an alias for `wset` field and it matches "Mem Usage" - column of taskmgr.exe. - - - **vms**: aka "Virtual Memory Size", this is the total amount of virtual - memory used by the process. - On UNIX it matches "top"'s VIRT column - (see `doc `__). - On Windows this is an alias for `pagefile` field and it matches - "Mem Usage" "VM Size" column of taskmgr.exe. - - - **shared**: *(Linux)* - memory that could be potentially shared with other processes. - This matches "top"'s SHR column - (see `doc `__). - - - **text** *(Linux, BSD)*: - aka TRS (text resident set) the amount of memory devoted to - executable code. This matches "top"'s CODE column - (see `doc `__). - - - **data** *(Linux, BSD)*: - aka DRS (data resident set) the amount of physical memory devoted to - other than executable code. It matches "top"'s DATA column - (see `doc `__). - - - **lib** *(Linux)*: the memory used by shared libraries. - - - **dirty** *(Linux)*: the number of dirty pages. - - For Windows fields rely on - `PROCESS_MEMORY_COUNTERS_EX `__ structure doc. - Example on Linux: - - >>> import psutil - >>> p = psutil.Process() - >>> p.memory_info() - pmem(rss=15491072, vms=84025344, shared=5206016, text=2555904, lib=0, data=9891840, dirty=0) - - .. versionchanged:: - 4.0.0 mutiple fields are returned, not only `rss` and `vms`. - - .. method:: memory_info_ex() - - Same as :meth:`memory_info` (deprecated). - - .. warning:: - deprecated in version 4.0.0; use :meth:`memory_info` instead. - - .. method:: memory_full_info() - - This method returns the same information as :meth:`memory_info`, plus, on - some platform (Linux, OSX, Windows), also provides additional metrics - (USS, PSS and swap). - The additional metrics provide a better representation of "effective" - process memory consumption (in case of USS) as explained in detail - `here `__. - It does so by passing through the whole process address. - As such it usually requires higher user privileges than - :meth:`memory_info` and is considerably slower. - On platforms where extra fields are not implented this simply returns the - same metrics as :meth:`memory_info`. - - - **uss** *(Linux, OSX, Windows)*: - aka "Unique Set Size", this is the memory which is unique to a process - and which would be freed if the process was terminated right now. - - - **pss** *(Linux)*: aka "Proportional Set Size", is the amount of memory - shared with other processes, accounted in a way that the amount is - divided evenly between the processes that share it. - I.e. if a process has 10 MBs all to itself and 10 MBs shared with - another process its PSS will be 15 MBs. - - - **swap** *(Linux)*: amount of memory that has been swapped out to disk. - - .. note:: - `uss` is probably the most representative metric for determining how - much memory is actually being used by a process. - It represents the amount of memory that would be freed if the process - was terminated right now. - - Example on Linux: - - >>> import psutil - >>> p = psutil.Process() - >>> p.memory_full_info() - pfullmem(rss=10199040, vms=52133888, shared=3887104, text=2867200, lib=0, data=5967872, dirty=0, uss=6545408, pss=6872064, swap=0) - >>> - - See also `scripts/procsmem.py `__ - for an example application. - - .. versionadded:: 4.0.0 - - .. method:: memory_percent(memtype="rss") - - Compare process memory to total physical system memory and calculate - process memory utilization as a percentage. - *memtype* argument is a string that dictates what type of process memory - you want to compare against. You can choose between the namedtuple field - names returned by :meth:`memory_info` and :meth:`memory_full_info` - (defaults to ``"rss"``). - - .. versionchanged:: 4.0.0 added `memtype` parameter. - - .. method:: memory_maps(grouped=True) - - Return process's mapped memory regions as a list of namedtuples whose - fields are variable depending on the platform. - This method is useful to obtain a detailed representation of process - memory usage as explained - `here `__ - (the most important value is "private" memory). - If *grouped* is ``True`` the mapped regions with the same *path* are - grouped together and the different memory fields are summed. If *grouped* - is ``False`` each mapped region is shown as a single entity and the - namedtuple will also include the mapped region's address space (*addr*) - and permission set (*perms*). - See `scripts/pmap.py `__ - for an example application. - - +---------------+--------------+---------+-----------+--------------+ - | Linux | OSX | Windows | Solaris | FreeBSD | - +===============+==============+=========+===========+==============+ - | rss | rss | rss | rss | rss | - +---------------+--------------+---------+-----------+--------------+ - | size | private | | anonymous | private | - +---------------+--------------+---------+-----------+--------------+ - | pss | swapped | | locked | ref_count | - +---------------+--------------+---------+-----------+--------------+ - | shared_clean | dirtied | | | shadow_count | - +---------------+--------------+---------+-----------+--------------+ - | shared_dirty | ref_count | | | | - +---------------+--------------+---------+-----------+--------------+ - | private_clean | shadow_depth | | | | - +---------------+--------------+---------+-----------+--------------+ - | private_dirty | | | | | - +---------------+--------------+---------+-----------+--------------+ - | referenced | | | | | - +---------------+--------------+---------+-----------+--------------+ - | anonymous | | | | | - +---------------+--------------+---------+-----------+--------------+ - | swap | | | | | - +---------------+--------------+---------+-----------+--------------+ - - >>> import psutil - >>> p = psutil.Process() - >>> p.memory_maps() - [pmmap_grouped(path='/lib/x8664-linux-gnu/libutil-2.15.so', rss=32768, size=2125824, pss=32768, shared_clean=0, shared_dirty=0, private_clean=20480, private_dirty=12288, referenced=32768, anonymous=12288, swap=0), - pmmap_grouped(path='/lib/x8664-linux-gnu/libc-2.15.so', rss=3821568, size=3842048, pss=3821568, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=3821568, referenced=3575808, anonymous=3821568, swap=0), - pmmap_grouped(path='/lib/x8664-linux-gnu/libcrypto.so.0.1', rss=34124, rss=32768, size=2134016, pss=15360, shared_clean=24576, shared_dirty=0, private_clean=0, private_dirty=8192, referenced=24576, anonymous=8192, swap=0), - pmmap_grouped(path='[heap]', rss=32768, size=139264, pss=32768, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=32768, referenced=32768, anonymous=32768, swap=0), - pmmap_grouped(path='[stack]', rss=2465792, size=2494464, pss=2465792, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=2465792, referenced=2277376, anonymous=2465792, swap=0), - ...] - >>> - - Availability: All platforms except OpenBSD and NetBSD. - - .. method:: children(recursive=False) - - Return the children of this process as a list of :Class:`Process` objects, - preemptively checking whether PID has been reused. If recursive is `True` - return all the parent descendants. - Example assuming *A == this process*: - :: - - A ─┠- │ - ├─ B (child) ─┠- │ └─ X (grandchild) ─┠- │ └─ Y (great grandchild) - ├─ C (child) - └─ D (child) - - >>> p.children() - B, C, D - >>> p.children(recursive=True) - B, X, Y, C, D - - Note that in the example above if process X disappears process Y won't be - returned either as the reference to process A is lost. - - .. method:: open_files() - - Return regular files opened by process as a list of namedtuples including - the following fields: - - - **path**: the absolute file name. - - **fd**: the file descriptor number; on Windows this is always ``-1``. - - **position** (*Linux*): the file (offset) position. - - **mode** (*Linux*): a string indicating how the file was opened, similarly - `open `__'s - ``mode`` argument. Possible values are ``'r'``, ``'w'``, ``'a'``, - ``'r+'`` and ``'a+'``. There's no distinction between files opened in - bynary or text mode (``"b"`` or ``"t"``). - - **flags** (*Linux*): the flags which were passed to the underlying - `os.open `__ C call - when the file was opened (e.g. - `os.O_RDONLY `__, - `os.O_TRUNC `__, - etc). - - >>> import psutil - >>> f = open('file.ext', 'w') - >>> p = psutil.Process() - >>> p.open_files() - [popenfile(path='/home/giampaolo/svn/psutil/setup.py', fd=3, position=0, mode='r', flags=32768), - popenfile(path='/var/log/monitd', fd=4, position=235542, mode='a', flags=33793)] - - .. warning:: - on Windows this is not fully reliable as due to some limitations of the - Windows API the underlying implementation may hang when retrieving - certain file handles. - In order to work around that psutil on Windows Vista (and higher) spawns - a thread and kills it if it's not responding after 100ms. - That implies that on Windows this method is not guaranteed to enumerate - all regular file handles (see full - `discussion `_). - - .. warning:: - on BSD this method can return files with a 'null' path due to a kernel - bug hence it's not reliable - (see `issue 595 `_). - - .. versionchanged:: - 3.1.0 no longer hangs on Windows. - - .. versionchanged:: - 4.1.0 new *position*, *mode* and *flags* fields on Linux. - - .. method:: connections(kind="inet") - - Return socket connections opened by process as a list of namedtuples. - To get system-wide connections use :func:`psutil.net_connections()`. - Every namedtuple provides 6 attributes: - - - **fd**: the socket file descriptor. This can be passed to - `socket.fromfd() `__ - to obtain a usable socket object. - This is only available on UNIX; on Windows ``-1`` is always returned. - - **family**: the address family, either `AF_INET - `__, - `AF_INET6 `__ - or `AF_UNIX `__. - - **type**: the address type, either `SOCK_STREAM - `__ or - `SOCK_DGRAM - `__. - - **laddr**: the local address as a ``(ip, port)`` tuple or a ``path`` - in case of AF_UNIX sockets. - - **raddr**: the remote address as a ``(ip, port)`` tuple or an absolute - ``path`` in case of UNIX sockets. - When the remote endpoint is not connected you'll get an empty tuple - (AF_INET) or ``None`` (AF_UNIX). - On Linux AF_UNIX sockets will always have this set to ``None``. - - **status**: represents the status of a TCP connection. The return value - is one of the :data:`psutil.CONN_* ` constants. - For UDP and UNIX sockets this is always going to be - :const:`psutil.CONN_NONE`. - - The *kind* parameter is a string which filters for connections that fit the - following criteria: - - +----------------+-----------------------------------------------------+ - | **Kind value** | **Connections using** | - +================+=====================================================+ - | "inet" | IPv4 and IPv6 | - +----------------+-----------------------------------------------------+ - | "inet4" | IPv4 | - +----------------+-----------------------------------------------------+ - | "inet6" | IPv6 | - +----------------+-----------------------------------------------------+ - | "tcp" | TCP | - +----------------+-----------------------------------------------------+ - | "tcp4" | TCP over IPv4 | - +----------------+-----------------------------------------------------+ - | "tcp6" | TCP over IPv6 | - +----------------+-----------------------------------------------------+ - | "udp" | UDP | - +----------------+-----------------------------------------------------+ - | "udp4" | UDP over IPv4 | - +----------------+-----------------------------------------------------+ - | "udp6" | UDP over IPv6 | - +----------------+-----------------------------------------------------+ - | "unix" | UNIX socket (both UDP and TCP protocols) | - +----------------+-----------------------------------------------------+ - | "all" | the sum of all the possible families and protocols | - +----------------+-----------------------------------------------------+ - - Example: - - >>> import psutil - >>> p = psutil.Process(1694) - >>> p.name() - 'firefox' - >>> p.connections() - [pconn(fd=115, family=, type=, laddr=('10.0.0.1', 48776), raddr=('93.186.135.91', 80), status='ESTABLISHED'), - pconn(fd=117, family=, type=, laddr=('10.0.0.1', 43761), raddr=('72.14.234.100', 80), status='CLOSING'), - pconn(fd=119, family=, type=, laddr=('10.0.0.1', 60759), raddr=('72.14.234.104', 80), status='ESTABLISHED'), - pconn(fd=123, family=, type=, laddr=('10.0.0.1', 51314), raddr=('72.14.234.83', 443), status='SYN_SENT')] - - .. method:: is_running() - - Return whether the current process is running in the current process list. - This is reliable also in case the process is gone and its PID reused by - another process, therefore it must be preferred over doing - ``psutil.pid_exists(p.pid)``. - - .. note:: - this will return ``True`` also if the process is a zombie - (``p.status() == psutil.STATUS_ZOMBIE``). - - .. method:: send_signal(signal) - - Send a signal to process (see - `signal module `__ - constants) preemptively checking whether PID has been reused. - On UNIX this is the same as ``os.kill(pid, sig)``. - On Windows only **SIGTERM**, **CTRL_C_EVENT** and **CTRL_BREAK_EVENT** - signals are supported and **SIGTERM** is treated as an alias for - :meth:`kill()`. - - .. versionchanged:: - 3.2.0 support for CTRL_C_EVENT and CTRL_BREAK_EVENT signals on Windows - was added. - - .. method:: suspend() - - Suspend process execution with **SIGSTOP** signal preemptively checking - whether PID has been reused. - On UNIX this is the same as ``os.kill(pid, signal.SIGSTOP)``. - On Windows this is done by suspending all process threads execution. - - .. method:: resume() - - Resume process execution with **SIGCONT** signal preemptively checking - whether PID has been reused. - On UNIX this is the same as ``os.kill(pid, signal.SIGCONT)``. - On Windows this is done by resuming all process threads execution. - - .. method:: terminate() - - Terminate the process with **SIGTERM** signal preemptively checking - whether PID has been reused. - On UNIX this is the same as ``os.kill(pid, signal.SIGTERM)``. - On Windows this is an alias for :meth:`kill`. - - .. method:: kill() - - Kill the current process by using **SIGKILL** signal preemptively - checking whether PID has been reused. - On UNIX this is the same as ``os.kill(pid, signal.SIGKILL)``. - On Windows this is done by using - `TerminateProcess `__. - - .. method:: wait(timeout=None) - - Wait for process termination and if the process is a children of the - current one also return the exit code, else ``None``. On Windows there's - no such limitation (exit code is always returned). If the process is - already terminated immediately return ``None`` instead of raising - :class:`NoSuchProcess`. If *timeout* is specified and process is still - alive raise :class:`TimeoutExpired` exception. It can also be used in a - non-blocking fashion by specifying ``timeout=0`` in which case it will - either return immediately or raise :class:`TimeoutExpired`. - To wait for multiple processes use :func:`psutil.wait_procs()`. - - -Popen class ------------ - -.. class:: Popen(*args, **kwargs) - - A more convenient interface to stdlib - `subprocess.Popen `__. - It starts a sub process and deals with it exactly as when using - `subprocess.Popen `__ - but in addition it also provides all the methods of - :class:`psutil.Process` class in a single interface. - For method names common to both classes such as - :meth:`send_signal() `, - :meth:`terminate() ` and - :meth:`kill() ` - :class:`psutil.Process` implementation takes precedence. - For a complete documentation refer to - `subprocess module documentation `__. - - .. note:: - - Unlike `subprocess.Popen `__ - this class preemptively checks whether PID has been reused on - :meth:`send_signal() `, - :meth:`terminate() ` and - :meth:`kill() ` - so that you can't accidentally terminate another process, fixing - http://bugs.python.org/issue6973. - - >>> import psutil - >>> from subprocess import PIPE - >>> - >>> p = psutil.Popen(["/usr/bin/python", "-c", "print('hello')"], stdout=PIPE) - >>> p.name() - 'python' - >>> p.username() - 'giampaolo' - >>> p.communicate() - ('hello\n', None) - >>> p.wait(timeout=2) - 0 - >>> - -Windows services -================ - -.. function:: win_service_iter() - - Return an iterator yielding a :class:`WindowsService` class instance for all - Windows services installed. - - .. versionadded:: 4.2.0 - - Availability: Windows - -.. function:: win_service_get(name) - - Get a Windows service by name, returning a :class:`WindowsService` instance. - Raise :class:`psutil.NoSuchProcess` if no service with such name exists. - - .. versionadded:: 4.2.0 - - Availability: Windows - -.. class:: WindowsService - - Represents a Windows service with the given *name*. This class is returned - by :func:`win_service_iter` and :func:`win_service_get` functions and it is - not supposed to be instantiated directly. - - .. method:: name() - - The service name. This string is how a service is referenced and can be - passed to :func:`win_service_get` to get a new :class:`WindowsService` - instance. - - .. method:: display_name() - - The service display name. The value is cached when this class is - instantiated. - - .. method:: binpath() - - The fully qualified path to the service binary/exe file as a string, - including command line arguments. - - .. method:: username() - - The name of the user that owns this service. - - .. method:: start_type() - - A string which can either be `"automatic"`, `"manual"` or `"disabled"`. - - .. method:: pid() - - The process PID, if any, else `None`. This can be passed to - :class:`Process` class to control the service's process. - - .. method:: status() - - Service status as a string, which may be either `"running"`, `"paused"`, - `"start_pending"`, `"pause_pending"`, `"continue_pending"`, - `"stop_pending"` or `"stopped"`. - - .. method:: description() - - Service long description. - - .. method:: as_dict() - - Utility method retrieving all the information above as a dictionary. - - .. versionadded:: 4.2.0 - - Availability: Windows - -Example code: - - >>> import psutil - >>> list(psutil.win_service_iter()) - [, - , - , - , - ...] - >>> s = psutil.win_service_get('alg') - >>> s.as_dict() - {'binpath': 'C:\\Windows\\System32\\alg.exe', - 'description': 'Provides support for 3rd party protocol plug-ins for Internet Connection Sharing', - 'display_name': 'Application Layer Gateway Service', - 'name': 'alg', - 'pid': None, - 'start_type': 'manual', - 'status': 'stopped', - 'username': 'NT AUTHORITY\\LocalService'} - -Constants -========= - -.. _const-oses: -.. data:: POSIX - WINDOWS - LINUX - OSX - FREEBSD - NETBSD - OPENBSD - BSD - SUNOS - - ``bool`` constants which define what platform you're on. E.g. if on Windows, - *WINDOWS* constant will be ``True``, all others will be ``False``. - - .. versionadded:: 4.0.0 - -.. _const-procfs_path: -.. data:: PROCFS_PATH - - The path of the /proc filesystem on Linux and Solaris (defaults to "/proc"). - You may want to re-set this constant right after importing psutil in case - your /proc filesystem is mounted elsewhere. - - Availability: Linux, Solaris - - .. versionadded:: 3.2.3 - .. versionchanged:: 3.4.2 also available on Solaris. - -.. _const-pstatus: -.. data:: STATUS_RUNNING - STATUS_SLEEPING - STATUS_DISK_SLEEP - STATUS_STOPPED - STATUS_TRACING_STOP - STATUS_ZOMBIE - STATUS_DEAD - STATUS_WAKE_KILL - STATUS_WAKING - STATUS_IDLE (OSX, FreeBSD) - STATUS_LOCKED (FreeBSD) - STATUS_WAITING (FreeBSD) - STATUS_SUSPENDED (NetBSD) - - A set of strings representing the status of a process. - Returned by :meth:`psutil.Process.status()`. - - .. versionadded:: 3.4.1 STATUS_SUSPENDED (NetBSD) - -.. _const-conn: -.. data:: CONN_ESTABLISHED - CONN_SYN_SENT - CONN_SYN_RECV - CONN_FIN_WAIT1 - CONN_FIN_WAIT2 - CONN_TIME_WAIT - CONN_CLOSE - CONN_CLOSE_WAIT - CONN_LAST_ACK - CONN_LISTEN - CONN_CLOSING - CONN_NONE - CONN_DELETE_TCB (Windows) - CONN_IDLE (Solaris) - CONN_BOUND (Solaris) - - A set of strings representing the status of a TCP connection. - Returned by :meth:`psutil.Process.connections()` (`status` field). - -.. _const-prio: -.. data:: ABOVE_NORMAL_PRIORITY_CLASS - BELOW_NORMAL_PRIORITY_CLASS - HIGH_PRIORITY_CLASS - IDLE_PRIORITY_CLASS - NORMAL_PRIORITY_CLASS - REALTIME_PRIORITY_CLASS - - A set of integers representing the priority of a process on Windows (see - `MSDN documentation `__). - They can be used in conjunction with - :meth:`psutil.Process.nice()` to get or set process priority. - - Availability: Windows - - .. versionchanged:: - 3.0.0 on Python >= 3.4 these constants are - `enums `__ - instead of a plain integer. - -.. _const-ioprio: -.. data:: IOPRIO_CLASS_NONE - IOPRIO_CLASS_RT - IOPRIO_CLASS_BE - IOPRIO_CLASS_IDLE - - A set of integers representing the I/O priority of a process on Linux. They - can be used in conjunction with :meth:`psutil.Process.ionice()` to get or set - process I/O priority. - *IOPRIO_CLASS_NONE* and *IOPRIO_CLASS_BE* (best effort) is the default for - any process that hasn't set a specific I/O priority. - *IOPRIO_CLASS_RT* (real time) means the process is given first access to the - disk, regardless of what else is going on in the system. - *IOPRIO_CLASS_IDLE* means the process will get I/O time when no-one else - needs the disk. - For further information refer to manuals of - `ionice `__ - command line utility or - `ioprio_get `__ - system call. - - Availability: Linux - - .. versionchanged:: - 3.0.0 on Python >= 3.4 thse constants are - `enums `__ - instead of a plain integer. - -.. _const-rlimit: -.. data:: RLIMIT_INFINITY - RLIMIT_AS - RLIMIT_CORE - RLIMIT_CPU - RLIMIT_DATA - RLIMIT_FSIZE - RLIMIT_LOCKS - RLIMIT_MEMLOCK - RLIMIT_MSGQUEUE - RLIMIT_NICE - RLIMIT_NOFILE - RLIMIT_NPROC - RLIMIT_RSS - RLIMIT_RTPRIO - RLIMIT_RTTIME - RLIMIT_RTPRIO - RLIMIT_SIGPENDING - RLIMIT_STACK - - Constants used for getting and setting process resource limits to be used in - conjunction with :meth:`psutil.Process.rlimit()`. See - `man prlimit `__ for further information. - - Availability: Linux - -.. _const-aflink: -.. data:: AF_LINK - - Constant which identifies a MAC address associated with a network interface. - To be used in conjunction with :func:`psutil.net_if_addrs()`. - - .. versionadded:: 3.0.0 - -.. _const-duplex: -.. data:: NIC_DUPLEX_FULL - NIC_DUPLEX_HALF - NIC_DUPLEX_UNKNOWN - - Constants which identifies whether a NIC (network interface card) has full or - half mode speed. NIC_DUPLEX_FULL means the NIC is able to send and receive - data (files) simultaneously, NIC_DUPLEX_FULL means the NIC can either send or - receive data at a time. - To be used in conjunction with :func:`psutil.net_if_stats()`. - - .. versionadded:: 3.0.0 - -Development guide -================= - -If you plan on hacking on psutil (e.g. want to add a new feature or fix a bug) -take a look at the -`development guide `_. + + + +.. ============================================================================ +.. Sponsors +.. ============================================================================ + +.. raw:: html + +
    + + +.. raw:: html + :file: _sponsors.html + +.. raw:: html + +
    + +.. ============================================================================ +.. TOC: hidden via CSS, needed by Sphinx for sidebar nav +.. ============================================================================ + +.. toctree:: + :maxdepth: 2 + :caption: Documentation + + Install + API overview + API Reference + FAQ + Performance + Recipes + +.. toctree:: + :maxdepth: 2 + :caption: Reference + + Shell equivalents + Stdlib equivalents + Glossary + Platform support + Migration + +.. toctree:: + :maxdepth: 2 + :caption: About + + Who uses psutil + Alternatives + Funding + Credits + +.. toctree:: + :maxdepth: 2 + :titlesonly: + :caption: Project + + Blog + Changelog + Timeline + Development guide + General Index diff --git a/docs/install.rst b/docs/install.rst new file mode 100644 index 0000000000..6927dde566 --- /dev/null +++ b/docs/install.rst @@ -0,0 +1,199 @@ +Install psutil +============== + +Linux, Windows, macOS (wheels) +------------------------------ + +Prebuilt wheels are distributed for these platforms, so a C compiler is not +required. Install psutil with: + +.. code-block:: none + + pip install psutil + +Inside a virtual environment you can also use +`uv `_: + +.. code-block:: none + + uv pip install psutil + +If no wheel is available for your platform or architecture, pip will build +psutil from source (see below). + +.. _install_from_source: + +Build psutil from source +------------------------ + +Building psutil from source requires a C compiler and the Python development +headers. On Linux, FreeBSD, NetBSD, OpenBSD and Solaris, the +`install-sysdeps.sh`_ script can install them for you: + +.. code-block:: none + + curl -fsSL https://raw.githubusercontent.com/giampaolo/psutil/master/scripts/internal/install-sysdeps.sh | sh + +Alternatively, install them manually as described below. + +Linux +^^^^^ + +Debian / Ubuntu: + +.. code-block:: none + + sudo apt-get install gcc python3-dev + +Red Hat / CentOS: + +.. code-block:: none + + sudo yum install gcc python3-devel + +Fedora: + +.. code-block:: none + + sudo dnf install gcc python3-devel + +Arch: + +.. code-block:: none + + sudo pacman -S gcc python + +Alpine: + +.. code-block:: none + + sudo apk add gcc python3-dev musl-dev linux-headers + +.. _install_windows: + +Windows +^^^^^^^ + +- To build psutil from source, install + `Microsoft C++ Build Tools `_ + with the **Desktop development with C++** option selected. +- MinGW is not supported. +- To clone psutil's Git repository and build or develop it locally, first + install `Git for Windows`_ and GNU Make. To install GNU Make, open PowerShell + and run: + + .. code-block:: none + + winget install --exact --id ezwinports.make + +- Close and reopen Git Bash, then follow :ref:`build_and_install`. + +macOS +^^^^^ + +Install the Xcode command line tools: + +.. code-block:: none + + xcode-select --install + +FreeBSD +^^^^^^^ + +.. code-block:: none + + pkg install python3 + +OpenBSD +^^^^^^^ + +.. code-block:: none + + export PKG_PATH=https://cdn.openbsd.org/pub/OpenBSD/`uname -r`/packages/`uname -m`/ + pkg_add -v python%3 + +NetBSD +^^^^^^ + +pkgsrc has no version-agnostic python3 package, so choose one of the available +Python 3 versions. For example: + +.. code-block:: none + + export PKG_PATH="https://cdn.netbsd.org/pub/pkgsrc/packages/NetBSD/`uname -m`/`uname -r`/All" + pkg_add -v pkgin + pkgin update + pkgin install python314 + +Solaris +^^^^^^^ + +.. code-block:: none + + pkg install developer/gcc + +If ``cc`` is unavailable, set ``CC=gcc`` when running the commands in +:ref:`build_and_install`. + +AIX +^^^ + +``install-sysdeps.sh`` has no AIX branch. Install a C compiler and the Python +development headers from the `AIX Toolbox`_. + +.. _build_and_install: + +Build and install +----------------- + +To build and install psutil from a Git checkout: + +.. code-block:: none + + git clone https://github.com/giampaolo/psutil.git + cd psutil + make install-sysdeps + make build + make install + +.. note:: + + By default C source files are compiled in parallel, one job per CPU, which + makes building from source 2x to 3.6x faster. Use + :envvar:`PSUTIL_BUILD_JOBS` to change the number of jobs. + +Troubleshooting +--------------- + +Install pip +^^^^^^^^^^^ + +Python installations normally include pip. If it is missing, first try: + +.. code-block:: none + + python3 -m ensurepip --upgrade + +Some OS-packaged Python installations do not include ``ensurepip``. There, +either install the pip package or download `get-pip.py`_ and run: + +.. code-block:: none + + python3 get-pip.py + +Permission errors +^^^^^^^^^^^^^^^^^ + +If you encounter permission errors, install psutil inside a virtual environment +instead of modifying the system Python installation: + +.. code-block:: none + + python3 -m venv .venv + source .venv/bin/activate + python -m pip install psutil + +.. _`AIX Toolbox`: https://www.ibm.com/support/pages/aix-toolbox-open-source-software-downloads-alpha +.. _`get-pip.py`: https://bootstrap.pypa.io/get-pip.py +.. _`Git for Windows`: https://git-scm.com/install/windows +.. _`install-sysdeps.sh`: https://github.com/giampaolo/psutil/blob/master/scripts/internal/install-sysdeps.sh diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 9bc67515c6..0000000000 --- a/docs/make.bat +++ /dev/null @@ -1,242 +0,0 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\psutil.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\psutil.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end diff --git a/docs/migration.rst b/docs/migration.rst new file mode 100644 index 0000000000..745abbcf7a --- /dev/null +++ b/docs/migration.rst @@ -0,0 +1,312 @@ +Migration guide +=============== + +This page summarises the breaking changes introduced in each major release and +shows the code changes required to upgrade. + +.. note:: + Minor and patch releases (e.g. 6.1.x, 7.1.x) do not contain breaking changes. + Only major releases are listed here. + +.. _migration-8.0: + +Migrating to 8.0 +----------------- + +Key breaking changes in 8.0: + +- :func:`process_iter` now pre-fetches values. +- :attr:`Process.info` is deprecated: use direct methods instead. +- Named tuple field order changed: use attribute access instead of positional + unpacking. +- Some return types are now enums instead of strings. +- :meth:`Process.memory_full_info` is deprecated: use + :meth:`Process.memory_footprint`. +- New :meth:`Process.memory_extras` method, returning extra platform-specific + memory metrics. +- New :attr:`Process.attrs`: :class:`frozenset` of valid attribute names; + ``process_iter(attrs=[])`` is deprecated. +- Python 3.6 and 3.7 dropped. +- Windows < 10 dropped. +- macOS 10.7 and 10.8 dropped. + +.. important:: + + Do not rely on positional unpacking of named tuples. Always use attribute + access (e.g. ``t.rss``). + +.. _migration-8.0-process-iter: + +process_iter(): p.info is deprecated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:func:`process_iter` now caches pre-fetched values internally, so normal method +calls can return them without using the :attr:`Process.info` dict. ``p.info`` +still works, but raises :exc:`DeprecationWarning`. + +.. code-block:: python + + import psutil + + # before + for p in psutil.process_iter(attrs=["name", "status"]): + print(p.info["name"], p.info["status"]) + + # after + for p in psutil.process_iter(attrs=["name", "status"]): + print(p.name(), p.status()) # return cached values, never raise + +When ``attrs`` are specified, the corresponding method calls return cached +values without extra syscalls. :exc:`AccessDenied` / :exc:`ZombieProcess` are +handled transparently by returning ``ad_value``. + +If you need a dict, use :meth:`Process.as_dict` instead of +:attr:`Process.info`. + +.. code-block:: python + + import psutil + + # before + for p in psutil.process_iter(attrs=["name", "status"]): + print(p.info) + + # after + attrs = ["name", "status"] + for p in psutil.process_iter(attrs=attrs): + print(p.as_dict(attrs)) # return cached values, never raise + +.. note:: + If ``"name"`` was pre-fetched via ``attrs``, ``p.name()`` returns + ``ad_value`` instead of raising :exc:`AccessDenied`. If you need the + exception, do not include the method in ``attrs``. + +.. _migration-8.0-namedtuples: + +Named tuple field order changed +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- :func:`cpu_times`: :field:`user`, :field:`system`, :field:`idle` fields + changed order on Linux, macOS and BSD. They are now always the first 3 fields + on all platforms, with platform-specific fields (e.g. :field:`nice`) + following. Positional access (e.g. ``cpu_times()[3]``) silently returns the + wrong field. + + .. code-block:: python + + # before + user, nice, system, idle = psutil.cpu_times() + + # after + t = psutil.cpu_times() + user, system, idle = t.user, t.system, t.idle + +- :meth:`Process.memory_info`: the returned named tuple changed size and field + order. + + - Linux: :field:`lib` and :field:`dirty` fields removed (they were always 0 + since Linux 2.6). Aliases returning 0 and emitting + :exc:`DeprecationWarning` are kept. + - macOS: :field:`pfaults` and :field:`pageins` removed with **no aliases**. + Use :meth:`Process.page_faults` instead. + - Windows: old fields were renamed: :field:`wset` → :field:`rss`, + :field:`peak_wset` → :field:`peak_rss`, :field:`pagefile` and + :field:`private` → :field:`vms`, :field:`peak_pagefile` → + :field:`peak_vms`, :field:`num_page_faults` → :meth:`Process.page_faults`. + The old names still work but raise :exc:`DeprecationWarning`. + :field:`paged_pool`, :field:`nonpaged_pool`, :field:`peak_paged_pool`, + :field:`peak_nonpaged_pool` moved to :meth:`Process.memory_extras`. + - BSD: a new :field:`peak_rss` field was added. + +- :func:`virtual_memory`: on Windows, new :field:`cached` and :field:`wired` + fields were added. + +cpu_times() interrupt renamed to irq on Windows +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The :field:`interrupt` field of :func:`cpu_times` on Windows was renamed to +:field:`irq` to match Linux and BSD. The old name still works but raises +:exc:`DeprecationWarning`. + +.. _migration-8.0-enums: + +Constants and fields are now enums +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These now yield enum members instead of plain ``str`` / ``int``, and the +matching module constants are members of the same enums: + +- :meth:`Process.status` → :class:`ProcessStatus` +- :field:`status` field of :meth:`Process.net_connections` and + :func:`net_connections` → :class:`ConnectionStatus` +- :meth:`Process.nice` on Windows → :class:`ProcessPriority` +- :field:`ioclass` field of :meth:`Process.ionice` → :class:`ProcessIOPriority` +- :data:`RLIMIT_* ` of :meth:`Process.rlimit` → + :class:`ProcessRlimit` + +They subclass :class:`enum.StrEnum` / :class:`enum.IntEnum`, so they compare +equal to the values they replace: ``p.status() == psutil.STATUS_RUNNING`` keeps +working. Only code inspecting :func:`repr` or :class:`type` needs updating. + +.. _migration-8.0-memory-full-info: + +memory_full_info() is deprecated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:meth:`Process.memory_full_info` is deprecated. Use +:meth:`Process.memory_footprint` instead; it returns the same fields +(:field:`uss`, :field:`pss` and :field:`swap`). + +.. _migration-8.0-memory-extras: + +New memory_extras() method +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +8.0 introduces a new :meth:`Process.memory_extras` method, returning extra +platform-specific memory metrics which complement :meth:`Process.memory_info`: + +- Linux: :field:`peak_rss`, :field:`peak_vms`, :field:`rss_anon`, + :field:`rss_file`, :field:`rss_shmem`, :field:`swap_anon`, :field:`hugetlb`. +- macOS: :field:`phys_footprint`, :field:`peak_footprint`. +- Windows: :field:`virtual`, :field:`peak_virtual`, :field:`paged_pool`, + :field:`nonpaged_pool`, :field:`peak_paged_pool`, + :field:`peak_nonpaged_pool`. + +.. _migration-8.0-attrs: + +New Process.attrs class attribute +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:attr:`Process.attrs` is a new :class:`frozenset` containing the valid +attribute names accepted by :meth:`Process.as_dict` and :func:`process_iter`. +It avoids creating a throwaway process just to discover them: + +.. code-block:: python + + # before + attrs = list(psutil.Process().as_dict().keys()) + + # after + attrs = psutil.Process.attrs + +It also makes it easy to pass all or a subset of attributes. +``process_iter(attrs=[])`` (empty list meaning "all") is now deprecated; use +:attr:`Process.attrs` instead: + +.. code-block:: python + + # all attrs + psutil.process_iter(attrs=psutil.Process.attrs) + + # all except connections + psutil.process_iter(attrs=psutil.Process.attrs - {"net_connections"}) + +Python 3.6 and 3.7 dropped +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The minimum version is now Python 3.8. + +.. _migration-8.0-windows: + +Windows < 10 dropped +^^^^^^^^^^^^^^^^^^^^^ + +Support for Windows Vista, 7, 8, 8.1 and their server counterparts (Server 2008 +to 2012 R2) was removed. The minimum version is now Windows 10 / Windows Server +2016. The last release supporting older versions is the 7.2.x series. See +:gh:`2893`. + +.. _migration-8.0-git-tags: + +Git tags renamed +^^^^^^^^^^^^^^^^^ + +Git tags were renamed from ``release-X.Y.Z`` to ``vX.Y.Z`` (e.g. +``release-7.2.2`` → ``v7.2.2``). Old tags remain for backward compatibility. If +your scripts or URLs reference psutil tags, update them to the new format. See +:gh:`2788`. + +------------------------------------------------------------------------------- + +.. _migration-7.0: + +Migrating to 7.0 +----------------- + +Process.memory_info_ex() removed +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``Process.memory_info_ex()``, deprecated since 4.0.0 in 2016, was removed. Use +:meth:`Process.memory_full_info` instead. + +.. code-block:: python + + # before + p.memory_info_ex() + + # after + p.memory_full_info() + +Python 2.7 dropped +^^^^^^^^^^^^^^^^^^^^ + +Python 2.7 is no longer supported. The last release supporting it is psutil +6.1.x: + +.. code-block:: bash + + pip2 install "psutil==6.1.*" + +------------------------------------------------------------------------------- + +.. _migration-6.0: + +Migrating to 6.0 +----------------- + +Process.connections() renamed +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:meth:`Process.connections` was renamed to :meth:`Process.net_connections` for +consistency with the system-level :func:`net_connections`. The old name raises +:exc:`DeprecationWarning` and will be removed in a future release: + +.. code-block:: python + + # before + p.connections() + p.connections(kind="tcp") + + # after + p.net_connections() + p.net_connections(kind="tcp") + +disk_partitions() lost two fields +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The :field:`maxfile` and :field:`maxpath` fields were removed from the named +tuple returned by :func:`disk_partitions`. Positional unpacking will break: + +.. code-block:: python + + # before (broken) + device, mountpoint, fstype, opts, maxfile, maxpath = part + + # after + device, mountpoint, fstype, opts = ( + part.device, part.mountpoint, part.fstype, part.opts + ) + +process_iter() no longer checks for PID reuse +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:func:`process_iter` no longer preemptively checks whether yielded PIDs have +been reused, making it ~20× faster. To verify that a process object is still +alive and refers to the same process, use :meth:`Process.is_running` +explicitly: + +.. code-block:: python + + for p in psutil.process_iter(["name"]): + if p.is_running(): + print(p.pid, p.name()) diff --git a/docs/performance.rst b/docs/performance.rst new file mode 100644 index 0000000000..3cb4af6302 --- /dev/null +++ b/docs/performance.rst @@ -0,0 +1,278 @@ +Performance +=========== + +This page describes how to use psutil efficiently. + +.. _perf-oneshot: + +Use oneshot() when reading multiple process attributes +------------------------------------------------------ + +If you're dealing with a single :class:`Process` instance and need to retrieve +multiple process attributes, use :meth:`Process.oneshot`. Each method call +issues a separate system call, but the OS often returns multiple attributes at +once, which :meth:`Process.oneshot` caches for subsequent calls. + +Slow: + +.. code-block:: python + + import psutil + + p = psutil.Process() + p.name() # syscall + p.cpu_times() # syscall + p.memory_info() # syscall + p.status() # syscall + +Fast: + +.. code-block:: python + + import psutil + + p = psutil.Process() + with p.oneshot(): + p.name() # one syscall, result cached + p.cpu_times() # from cache + p.memory_info() # from cache + p.status() # from cache + +The speed improvement depends on the platform and on how many attributes you +read. On Linux the gain is typically around 1.5x–2x; on Windows it can be much +higher. As a rule of thumb: if you read more than one attribute from the same +process, use :meth:`Process.oneshot`. + +.. _perf-process-iter: + +Use process_iter() with an attrs list +-------------------------------------- + +If you iterate over multiple PIDs, always use :func:`process_iter`. It accepts +an ``attrs`` argument that pre-fetches only the requested attributes in a +single pass, minimizing system calls by fetching multiple attributes at once. +This is faster than calling individual methods in a loop. + +Slow: + +.. code-block:: python + + import psutil + + for p in psutil.process_iter(): + try: + print(p.pid, p.name(), p.status()) + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + +Fast: + +.. code-block:: python + + import psutil + + for p in psutil.process_iter(["name", "status"]): + print(p.pid, p.name(), p.status()) # return cached values, never raise + +:func:`process_iter(attrs=...) ` is effectively equivalent +to using :meth:`Process.oneshot` on each process. Using :func:`process_iter` +also saves you from **race conditions** (e.g. if a process disappears while +iterating), since :exc:`NoSuchProcess` and :exc:`AccessDenied` exceptions are +handled internally. A typical use case is to fetch all process attrs except the +slow ones (see :ref:`perf-api-speed` table below): + +.. code-block:: python + + import psutil + + for p in psutil.process_iter(psutil.Process.attrs - {"memory_footprint", "memory_maps"}): + ... + +.. _perf-oneshot-methods: + +Methods sped up by oneshot() +---------------------------- + +Here's a list of method groups for each platform which can benefit from +:meth:`Process.oneshot`. Methods in each group (in the same comma-separated +list) share the same underlying system call. + +The *speedup* represents the estimated gain when all listed methods are called +together (best case), as measured by :src:`scripts/internal/bench_oneshot.py`. + +Additionally, some methods are computed from other methods, so on every +platform :meth:`Process.oneshot` also speeds up :meth:`~Process.cpu_percent` +(from :meth:`~Process.cpu_times`), :meth:`~Process.memory_percent` (from +:meth:`~Process.memory_info`), :meth:`~Process.parent` and +:meth:`~Process.parents` (from :meth:`~Process.ppid`) and, on POSIX, +:meth:`~Process.username` (from :meth:`~Process.uids`). + +Linux +""""" + +* :meth:`~Process.cpu_num`, :meth:`~Process.cpu_times`, + :meth:`~Process.create_time`, :meth:`~Process.name`, + :meth:`~Process.page_faults`, :meth:`~Process.ppid`, + :meth:`~Process.status`, :meth:`~Process.terminal` + +* :meth:`~Process.gids`, :meth:`~Process.memory_extras`, + :meth:`~Process.num_ctx_switches`, :meth:`~Process.num_threads`, + :meth:`~Process.uids` + +* :meth:`~Process.memory_footprint`, :meth:`~Process.memory_maps` + +*Speedup: +1.7×* + +Windows +""""""" + +* :meth:`~Process.cpu_times`, :meth:`~Process.io_counters`, + :meth:`~Process.memory_info`, :meth:`~Process.memory_extras`, + :meth:`~Process.num_ctx_switches`, :meth:`~Process.num_handles`, + :meth:`~Process.num_threads`, :meth:`~Process.page_faults`, + :meth:`~Process.status` + +* :meth:`~Process.exe`, :meth:`~Process.name` + +Some of these first try a faster dedicated call, and only use the shared one if +it raises :exc:`AccessDenied`. The second figure is for such processes. + +*Speedup: +1.8× / +6.5×* + +macOS +""""" + +* :meth:`~Process.cpu_times`, :meth:`~Process.memory_info`, + :meth:`~Process.num_ctx_switches`, :meth:`~Process.num_threads`, + :meth:`~Process.page_faults` + +* :meth:`~Process.create_time`, :meth:`~Process.gids`, :meth:`~Process.name`, + :meth:`~Process.ppid`, :meth:`~Process.status`, :meth:`~Process.terminal`, + :meth:`~Process.uids` + +*Speedup: +1.6×* + +BSD +""" + +* :meth:`~Process.cpu_num`, :meth:`~Process.cpu_times`, + :meth:`~Process.create_time`, :meth:`~Process.gids`, + :meth:`~Process.io_counters`, :meth:`~Process.memory_info`, + :meth:`~Process.name`, :meth:`~Process.nice`, + :meth:`~Process.num_ctx_switches`, :meth:`~Process.page_faults`, + :meth:`~Process.ppid`, :meth:`~Process.status`, :meth:`~Process.terminal`, + :meth:`~Process.uids` + +*Speedup: +2.7×* + +.. _perf-oneshot-bench: + +Measuring oneshot() speedup +--------------------------- + +:src:`scripts/internal/bench_oneshot.py` measures :meth:`Process.oneshot` +speedup. It also shows which APIs share the same internal kernel routines. E.g. +on Linux: + +.. code-block:: none + + $ python3 scripts/internal/bench_oneshot.py --times 10000 + 17 methods pre-fetched by oneshot() on platform 'linux' (10,000 times, psutil 8.0.0): + + cpu_num + cpu_percent + cpu_times + gids + memory_extras + memory_info + memory_percent + name + num_ctx_switches + num_threads + page_faults + parent + ppid + status + terminal + uids + username + + regular: 2.600 secs + oneshot: 1.499 secs + speedup: +1.73x + +.. _perf-api-speed: + +Measuring APIs speed +-------------------- + +:src:`scripts/internal/print_api_speed.py` shows the relative cost of each API +call. This helps you understand which operations are more expensive. E.g. on +Linux: + +.. code-block:: none + + $ python3 scripts/internal/print_api_speed.py + SYSTEM APIS NUM CALLS SECONDS + ------------------------------------------------- + getloadavg 300 0.00013 + heap_info 300 0.00028 + heap_trim 300 0.00039 + cpu_count 300 0.00061 + disk_usage 300 0.00066 + pid_exists 300 0.00235 + users 300 0.00455 + net_io_counters 300 0.00550 + cpu_times 300 0.00667 + boot_time 300 0.00700 + cpu_percent 300 0.00766 + net_if_stats 300 0.00783 + virtual_memory 300 0.00834 + cpu_times_percent 300 0.00885 + net_if_addrs 300 0.01157 + cpu_stats 300 0.01208 + swap_memory 300 0.01558 + disk_partitions 300 0.01664 + disk_io_counters 300 0.02204 + sensors_battery 300 0.02995 + pids 300 0.05295 + cpu_count (cores) 300 0.06943 + process_iter (all) 300 0.08486 + cpu_freq 300 0.18987 + sensors_fans 300 0.74027 + net_connections 161 2.00690 + sensors_temperatures 100 2.00742 + + PROCESS APIS NUM CALLS SECONDS + ------------------------------------------------- + exe 300 0.00017 + create_time 300 0.00020 + nice 300 0.00025 + ionice 300 0.00041 + cwd 300 0.00052 + cpu_affinity 300 0.00059 + num_fds 300 0.00097 + memory_info 300 0.00201 + cmdline 300 0.00222 + io_counters 300 0.00226 + cpu_num 300 0.00242 + status 300 0.00242 + terminal 300 0.00243 + name 300 0.00249 + page_faults 300 0.00258 + memory_percent 300 0.00259 + cpu_times 300 0.00272 + threads 300 0.00278 + num_threads 300 0.00278 + gids 300 0.00296 + num_ctx_switches 300 0.00299 + uids 300 0.00311 + cpu_percent 300 0.00346 + net_connections 300 0.00373 + open_files 300 0.00378 + memory_extras 300 0.00398 + username 300 0.00500 + ppid 300 0.00556 + environ 300 0.01176 + memory_footprint 300 0.02218 + memory_maps 300 0.27158 diff --git a/docs/platform.rst b/docs/platform.rst new file mode 100644 index 0000000000..cb6676aaf6 --- /dev/null +++ b/docs/platform.rst @@ -0,0 +1,243 @@ +Platform support +================ + +Python +^^^^^^ + +.. list-table:: + :class: wide-table + :header-rows: 1 + + * - Feature + - Support + - Notes + * - Minimum Python version + - 3.8 + - last version supporting 3.6 / 3.7 is + `psutil 7.2.2 `_ (Jan 2026) + * - PyPy + - yes + - not tested on CI + * - Free-threaded Python + - yes + - ``cp314t`` wheels are published + * - Stable ABI (abi3) + - yes + - ``cp38-abi3`` wheels support CPython 3.8+ + * - Inline type hints + - yes + - + * - Sub-interpreters + - partial + - legacy shared-GIL sub-interpreters work; ``concurrent.interpreters`` + does not (:gh:`2576`) + * - PEP 561 (``py.typed``) + - no + - + * - Python 2.7 + - no + - last version supporting it is + `psutil 6.1.1 `_ (Dec 2024) + +Operating systems +^^^^^^^^^^^^^^^^^ + +.. list-table:: + :class: wide-table + :header-rows: 1 + + * - Platform + - Minimum version + - Released + - Enforcement + - CI coverage + * - Linux + - 2.6.13 (soft) + - 2005 + - graceful fallbacks; no hard check + - yes + * - Windows + - 10 + - 2015 + - hard check at import and build time + - yes + * - macOS + - 10.9 (Mavericks) + - 2013 + - graceful fallbacks; no hard check + - yes + * - FreeBSD + - 12.0 + - 2018 + - graceful fallbacks via ``#if __FreeBSD_version`` + - yes + * - NetBSD + - 5.0 + - 2009 + - graceful fallbacks via ``#if __NetBSD_Version__`` + - yes + * - OpenBSD + - unknown + - + - + - yes + * - SunOS / Solaris + - 11 + - 2011 + - + - memleak tests only + * - AIX + - unknown + - + - + - no + +Except where a hard minimum is enforced, older releases may also work but are +not guaranteed to be supported. + +The minimum above is what the source builds against. Prebuilt macOS wheels +target 10.15, so 10.9 to 10.14 need to build from source. + +Architectures +^^^^^^^^^^^^^ + +.. list-table:: + :class: wide-table + :header-rows: 1 + + * - Architecture + - CI coverage + - Prebuilt wheels + - Manual testing + * - x86_64 + - Linux, Windows, macOS + - Linux, Windows, macOS + - + * - aarch64 / ARM64 + - Linux, Windows, macOS + - Linux, Windows, macOS + - + * - ppc64le + - + - Linux + - Debian 13 + * - s390x + - + - Linux + - + * - i686 + - + - + - Debian 13 + * - ppc64 (big endian) + - + - + - Debian 14 + * - riscv64 + - + - + - Debian 13 + * - sparc64 + - + - + - Debian 14 + +Occasional manual testing is done on the +`GCC compile farm `_, which provides free shell +access to uncommon hardware. + +On architectures without prebuilt wheels, psutil can be installed from source +(see :ref:`install_from_source`): + +.. code-block:: bash + + pip install psutil --no-binary psutil + +Linux wheels are published for both glibc (manylinux) and musl. The musl ones +cover x86_64 and aarch64 only. + +Support history +^^^^^^^^^^^^^^^ + +.. list-table:: + :class: wide-table + :header-rows: 1 + + * - Version + - Date + - Change + * - :pypi:`8.0.0` + - + - add wheels for Linux ppc64le and s390x architectures + * - :pypi:`8.0.0` + - + - drop Python 3.6 and 3.7 + * - :pypi:`8.0.0` + - + - drop PyPy older than 7.3.14 on Windows + * - :pypi:`8.0.0` + - + - drop Windows Vista, 7, 8 and 8.1 (+ Server 2008 to 2012 R2) + * - :pypi:`8.0.0` + - + - drop Intel wheel support for macOS < 10.15 + * - :pypi:`8.0.0` + - + - drop wheels for free-threaded Python 3.13 + * - :pypi:`7.2.0` + - 2025-12 + - drop wheels for Linux musl + * - :pypi:`7.1.2` + - 2025-10 + - drop wheels for free-threaded Python + * - :pypi:`7.1.2` + - 2025-10 + - drop wheels for 32-bit Python (Linux and Windows) + * - :pypi:`7.1.1` + - 2025-10 + - drop SunOS 10 + * - :pypi:`7.1.0` + - 2025-09 + - drop FreeBSD 8 + * - :pypi:`7.0.0` + - 2025-02 + - drop Python 2.7 + * - :pypi:`5.9.6` + - 2023-10 + - drop Python 3.4 and 3.5 + * - :pypi:`5.9.1` + - 2022-05 + - drop Python 2.6 + * - :pypi:`5.9.0` + - 2021-12 + - add MidnightBSD + * - :pypi:`5.8.0` + - 2020-12 + - add PyPy2 on Windows + * - :pypi:`5.7.1` + - 2020-07 + - add Windows Nano + * - :pypi:`5.7.0` + - 2020-02 + - drop Windows XP & Windows Server 2003 + * - :pypi:`5.7.0` + - 2020-02 + - add PyPy3 on Windows + * - :pypi:`5.4.0` + - 2017-11 + - add AIX + * - :pypi:`3.4.1` + - 2016-01 + - add NetBSD + * - :pypi:`3.3.0` + - 2015-11 + - add OpenBSD + * - :pypi:`1.0.0` + - 2013-07 + - add Solaris + * - :pypi:`0.1.1` + - 2009-03 + - add FreeBSD + * - :pypi:`0.1.0` + - 2009-01 + - add Linux, Windows, macOS diff --git a/docs/recipes.rst b/docs/recipes.rst new file mode 100644 index 0000000000..249e370d46 --- /dev/null +++ b/docs/recipes.rst @@ -0,0 +1,550 @@ +Recipes +======= + +A collection of standalone, copy-paste solutions to specific problems. Each +recipe focuses on a single problem and provides a minimal solution which can be +adapted to real-world code. The examples are intentionally short and avoid +unnecessary abstractions so that the underlying psutil APIs are easy to +understand. Most of them are not meant to be used in production. + +Processes +--------- + +Finding processes +^^^^^^^^^^^^^^^^^ + +.. _recipe_find_process_by_name: + +Find process by name: + +.. code-block:: python + + import psutil + + def find_procs_by_name(name): + ls = [] + for p in psutil.process_iter(["name"]): + if p.name() == name: + ls.append(p) + return ls + +------------------------------------------------------------------------------- + +A bit more advanced, check string against :meth:`Process.name`, +:meth:`Process.exe` and :meth:`Process.cmdline`: + +.. code-block:: python + + import os, psutil + + def find_procs_by_name_ex(name): + ls = [] + for p in psutil.process_iter(["name", "exe", "cmdline"]): + if ( + name == p.name() + or (p.exe() and os.path.basename(p.exe()) == name) + or (p.cmdline() and p.cmdline()[0] == name) + ): + ls.append(p) + return ls + +------------------------------------------------------------------------------- + +Find the process listening on a given TCP port: + +.. code-block:: python + + import psutil + + def find_proc_by_port(port): + for proc in psutil.process_iter(): + try: + cons = proc.net_connections(kind="tcp") + except psutil.Error: + pass + else: + for conn in cons: + if conn.laddr.port == port and conn.status == psutil.CONN_LISTEN: + return proc + return None + +------------------------------------------------------------------------------- + +Find all processes that have an active connection to a given remote IP: + +.. code-block:: python + + import psutil + + def find_procs_by_remote_host(host): + ls = [] + for proc in psutil.process_iter(): + try: + cons = proc.net_connections(kind="inet") + except psutil.Error: + pass + else: + for conn in cons: + if conn.raddr and conn.raddr.ip == host: + ls.append(proc) + return ls + +------------------------------------------------------------------------------- + +Find all processes that have a given file open (useful on Windows): + +.. code-block:: python + + import psutil + + def find_procs_using_file(path): + ls = [] + for p in psutil.process_iter(["open_files"]): + for f in p.open_files() or []: + if f.path == path: + ls.append(p) + break + return ls + +Filtering and sorting processes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Processes owned by user: + +.. code-block:: python + + import getpass, psutil + + def procs_by_user(user=None): + if user is None: + user = getpass.getuser() + return [ + (p.pid, p.name()) + for p in psutil.process_iter(["name", "username"]) + if p.username() == user + ] + +------------------------------------------------------------------------------- + +Processes using log files: + +.. code-block:: pycon + + >>> for p in psutil.process_iter(["name", "open_files"]): + ... for file in p.open_files() or []: + ... if file.path.endswith(".log"): + ... print("{:<5} {:<10} {}".format(p.pid, p.name()[:10], file.path)) + ... + 1510 upstart /home/giampaolo/.cache/upstart/unity-settings-daemon.log + 2174 nautilus /home/giampaolo/.local/share/gvfs-metadata/home-ce08efac.log + 2650 chrome /home/giampaolo/.config/google-chrome/Default/data_reduction_proxy_leveldb/000003.log + +------------------------------------------------------------------------------- + +Processes consuming more than 500M of memory: + +.. code-block:: python + + import psutil + + def procs_by_memory(min_bytes=500 * 1024 * 1024): + return [ + (p.pid, p.name(), p.memory_info().rss) + for p in psutil.process_iter(["name", "memory_info"]) + if p.memory_info().rss > min_bytes + ] + +------------------------------------------------------------------------------- + +Top N processes by :term:`cumulative ` CPU time: + +.. code-block:: python + + import psutil + + def top_cpu_procs(n=3): + procs = sorted( + psutil.process_iter(["name", "cpu_times"]), + key=lambda p: sum(p.cpu_times()[:2]), + ) + return [(p.pid, p.name(), sum(p.cpu_times())) for p in procs[-n:]] + +------------------------------------------------------------------------------- + +Top N processes by :term:`cumulative ` disk read + write +bytes (similar to ``iotop``): + +.. code-block:: python + + import psutil + + def top_io_procs(n=5): + procs = [] + for p in psutil.process_iter(["io_counters"]): + io = p.io_counters() + procs.append((io.read_bytes + io.write_bytes, p)) + procs.sort(key=lambda x: x[0], reverse=True) + return procs[:n] + +------------------------------------------------------------------------------- + +Top N processes by open file descriptors (useful for diagnosing fd leaks): + +.. code-block:: python + + import psutil + + def top_open_files(n=5): + procs = [] + for p in psutil.process_iter(["num_fds"]): + procs.append((p.num_fds(), p)) + procs.sort(key=lambda x: x[0], reverse=True) + return procs[:n] + +Monitoring processes +^^^^^^^^^^^^^^^^^^^^ + +Periodically monitor CPU and memory usage of a process using +:meth:`Process.oneshot` for efficiency: + +.. code-block:: python + + import time, psutil + + def monitor(pid, interval=1): + p = psutil.Process(pid) + while p.is_running(): + with p.oneshot(): + cpu = p.cpu_percent() + mem = p.memory_info().rss + print("cpu={:<6} mem={}".format(str(cpu) + "%", mem / 1024 / 1024)) + time.sleep(interval) + +.. code-block:: none + + cpu=4.2% mem=23.4M + cpu=3.1% mem=23.5M + +Controlling processes +^^^^^^^^^^^^^^^^^^^^^ + +.. _recipe_kill_proc_tree: + +Kill a process tree (including grandchildren): + +.. code-block:: python + + import os, signal, psutil + + def kill_proc_tree( + pid, + sig=signal.SIGTERM, + include_parent=True, + timeout=None, + on_terminate=None, + ): + """Kill a process tree (including grandchildren) with signal + "sig" and return a (gone, still_alive) tuple. + "on_terminate", if specified, is a callback function which is + called as soon as a child terminates. + """ + assert pid != os.getpid(), "I won't kill myself!" + parent = psutil.Process(pid) + children = parent.children(recursive=True) + # Reverse the list so that descendants are killed before + # their ancestors (bottom-up order). ``children()`` returns + # processes in top-down order; reversing ensures a grandchild + # is terminated before its parent. + children.reverse() + if include_parent: + children.append(parent) + for p in children: + try: + p.send_signal(sig) + except psutil.NoSuchProcess: + pass + gone, alive = psutil.wait_procs( + children, timeout=timeout, callback=on_terminate + ) + return (gone, alive) + +On Unix, if you started the subprocess with ``subprocess.Popen`` you can often +use the stdlib ``os.killpg()`` instead of this recipe. Create the process with +``process_group=0`` so that it gets its own process group, then call +``os.killpg(pgid, sig)``. This is simpler, does not require psutil, and still +cleans up all descendants even when an intermediate process has exited:: + + import os + import signal + import subprocess + + proc = subprocess.Popen(["cmd", "arg1"], process_group=0) + # ... later: + os.killpg(proc.pid, signal.SIGTERM) + +This approach does not work on Windows (``os.killpg`` is not available) and it +only works for PIDs that you started yourself as a new process group. For +arbitrary PIDs, use the psutil recipe above. + +------------------------------------------------------------------------------- + +Terminate a process gracefully, falling back to ``SIGKILL`` if it does not exit +within the timeout: + +.. code-block:: python + + import psutil + + def graceful_kill(pid, timeout=3): + p = psutil.Process(pid) + p.terminate() + try: + p.wait(timeout=timeout) + except psutil.TimeoutExpired: + p.kill() + +------------------------------------------------------------------------------- + +Temporarily pause and resume a process using a context manager: + +.. code-block:: python + + import contextlib, psutil + + @contextlib.contextmanager + def suspended(pid): + p = psutil.Process(pid) + p.suspend() + try: + yield p + finally: + p.resume() + + # usage + with suspended(pid): + pass # process is paused here + +------------------------------------------------------------------------------- + +CPU throttle: limit a process's CPU usage to a target percentage by alternating +:meth:`Process.suspend` and :meth:`Process.resume`: + +.. code-block:: python + + import time, psutil + + def throttle(pid, max_cpu_percent=50, interval=0.1): + """Slow down a process so it uses at most max_cpu_percent% CPU.""" + p = psutil.Process(pid) + while p.is_running(): + cpu = p.cpu_percent(interval=interval) + if cpu > max_cpu_percent: + p.suspend() + time.sleep(interval * cpu / max_cpu_percent) + p.resume() + +------------------------------------------------------------------------------- + +Restart a process automatically if it dies: + +.. code-block:: python + + import subprocess, time, psutil + + def watchdog(cmd, max_restarts=None, interval=1): + """Run cmd as a persistent process. Restart on failure, optionally + with a max restarts. Logs start, exit, and restart events. + """ + restarts = 0 + while True: + proc = subprocess.Popen(cmd) + p = psutil.Process(proc.pid) + print(f"started PID {p.pid}") + proc.wait() + + if proc.returncode == 0: + # success + print(f"PID {p.pid} exited cleanly") + break + + # failure + restarts += 1 + print(f"PID {p.pid} died, restarting ({restarts})") + + if max_restarts is not None and restarts > max_restarts: + print("max restarts reached, giving up") + break + + time.sleep(interval) + + + if __name__ == "__main__": + watchdog(["python3", "script.py"]) + +System +------ + +All APIs returning amounts (memory, disk, network I/O) express them in bytes. +The examples below use :func:`psutil.bytes2human` to convert them to a +human-readable string. + +CPU +^^^ + +Print real-time CPU usage percentage: + +.. code-block:: python + + import psutil + + while True: + print("CPU: {}%".format(psutil.cpu_percent(interval=1))) + +.. code-block:: none + + CPU: 2.1% + CPU: 1.4% + CPU: 0.9% + +Memory +^^^^^^ + +.. _recipe_swap_activity: + +Show real-time swap activity *(Linux, BSD)*. ``sout`` (:term:`swap-out`) is the +key metric: a non-zero and growing rate means the OS is moving memory from RAM +to disk because RAM is full. ``sin`` (:term:`swap-in`) alone is not alarming; +it just means the system is moving previously evicted pages back into RAM. High +``sin`` and ``sout`` together may indicate heavy swapping (:term:`thrashing`). + +.. code-block:: python + + import psutil, time + + def swap_activity(interval=1): + before = psutil.swap_memory() + while True: + time.sleep(interval) + after = psutil.swap_memory() + sin = after.sin - before.sin + sout = after.sout - before.sout + print("swap-in={}/s swap-out={}/s used={}%".format( + psutil.bytes2human(sin), psutil.bytes2human(sout), after.percent)) + before = after + +.. code-block:: none + + swap-in=0.0B/s swap-out=0.0B/s used=23% + swap-in=0.0B/s swap-out=1.2M/s used=24% + +Disks +^^^^^ + +.. _recipe_disk_io: + +Show real-time disk I/O: + +.. code-block:: python + + import psutil, time + + def disk_io(): + while True: + before = psutil.disk_io_counters() + time.sleep(1) + after = psutil.disk_io_counters() + r = after.read_bytes - before.read_bytes + w = after.write_bytes - before.write_bytes + print("Read: {}/s, Write: {}/s".format( + psutil.bytes2human(r), + psutil.bytes2human(w)) + ) + +.. code-block:: none + + Read: 1.2M/s, Write: 256.0K/s + Read: 0.0B/s, Write: 128.0K/s + +------------------------------------------------------------------------------- + +.. _recipe_disk_io_percent: + +Show real-time disk utilization percentage *(Linux, FreeBSD)*: + +.. code-block:: python + + import psutil, time + + def disk_io_percent(interval=1): + while True: + before = psutil.disk_io_counters() + time.sleep(interval) + after = psutil.disk_io_counters() + busy_ms = after.busy_time - before.busy_time + util = min(busy_ms / (interval * 1000) * 100, 100) + print("Disk: {:.1f}%".format(util)) + +.. code-block:: none + + Disk: 3.2% + Disk: 78.5% + +Network +^^^^^^^ + +Show real-time network I/O per interface: + +.. code-block:: python + + import psutil, time + + def net_io(): + while True: + before = psutil.net_io_counters(pernic=True) + time.sleep(1) + after = psutil.net_io_counters(pernic=True) + for iface in after: + s = after[iface].bytes_sent - before[iface].bytes_sent + r = after[iface].bytes_recv - before[iface].bytes_recv + print( + "{:<10} sent={:<10} recv={}".format( + iface, + psutil.bytes2human(s) + "/s", + psutil.bytes2human(r) + "/s" + ) + ) + print() + +.. code-block:: none + + lo sent=0.0B/s recv=0.0B/s + eth0 sent=12.3K/s recv=45.6K/s + +------------------------------------------------------------------------------- + +List all active TCP connections with their status: + +.. code-block:: python + + import psutil + + def netstat(): + templ = "{:<20} {:<20} {:<13} {:<6}" + print(templ.format("Local", "Remote", "Status", "PID")) + for conn in psutil.net_connections(kind="tcp"): + laddr = "{}:{}".format(conn.laddr.ip, conn.laddr.port) + raddr = ( + "{}:{}".format(conn.raddr.ip, conn.raddr.port) + if conn.raddr + else "-" + ) + print(templ.format(laddr, raddr, conn.status, conn.pid or "")) + +.. code-block:: none + + Local Remote Status PID + :::1716 - LISTEN 223441 + 127.0.0.1:631 - LISTEN + 10.0.0.4:45278 20.222.111.74:443 ESTABLISHED 437213 + 10.0.0.4:40130 172.14.148.135:443 ESTABLISHED + 0.0.0.0:22 - LISTEN 723345 diff --git a/docs/shell-equivalents.rst b/docs/shell-equivalents.rst new file mode 100644 index 0000000000..d3ad4bbd8c --- /dev/null +++ b/docs/shell-equivalents.rst @@ -0,0 +1,580 @@ +Shell equivalents +================= + +This page maps psutil's Python API to the equivalent native terminal commands +on each platform. This is useful for understanding what psutil replaces and for +cross-checking results. + +.. seealso:: + - :doc:`stdlib-equivalents` + - :doc:`alternatives` + +System-wide functions +--------------------- + +CPU +~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil function + - Linux + - macOS + - BSD + - Windows + * - :func:`cpu_percent` + - ``top`` + - same + - same + - + * - :func:`cpu_count(logical=True) ` + - ``nproc`` + - ``sysctl hw.logicalcpu`` + - ``sysctl hw.ncpu`` + - + * - :func:`cpu_count(logical=False) ` + - ``lscpu | grep '^Core(s)'`` + - ``sysctl hw.physicalcpu`` + - + - + * - :func:`cpu_times(percpu=False) ` + - ``cat /proc/stat | grep '^cpu\s'`` + - + - ``systat -vmstat`` + - + * - :func:`cpu_times(percpu=True) ` + - ``cat /proc/stat | grep '^cpu'`` + - + - ``systat -vmstat`` + - + * - :func:`cpu_times_percent(percpu=False) ` + - ``mpstat`` + - + - + - + * - :func:`cpu_times_percent(percpu=True) ` + - ``mpstat -P ALL`` + - + - + - + * - :func:`cpu_freq` + - ``cpufreq-info``, ``lscpu | grep "MHz"`` + - ``sysctl hw.cpufrequency`` + - ``sysctl dev.cpu.0.freq`` + - ``systeminfo`` + * - :func:`cpu_stats` + - + - + - ``vmstat -s``, ``sysctl vm.stats.sys`` + - + * - :func:`getloadavg` + - ``uptime`` + - same + - same + - + +Memory +~~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil function + - Linux + - macOS + - BSD + - Windows + * - :func:`virtual_memory` + - ``free``, ``vmstat``, ``cat /proc/meminfo`` + - ``vm_stat`` + - ``vmstat -s``, ``sysctl vm.stats`` + - ``systeminfo`` + * - :func:`swap_memory` + - ``free``, ``vmstat``, ``swapon`` + - ``sysctl vm.swapusage`` + - ``vmstat -s``, ``swapinfo`` + - + * - :func:`heap_info` + - + - + - + - + * - :func:`heap_trim` + - + - + - + - + +Disks +~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil function + - Linux + - macOS + - BSD + - Windows + * - :func:`disk_usage` + - ``df`` + - same + - same + - ``fsutil volume diskfree C:\`` + * - :func:`disk_partitions` + - ``mount``, ``findmnt`` + - ``mount`` + - ``mount`` + - + * - :func:`disk_io_counters` + - ``iostat -dx`` + - ``iostat`` + - ``iostat -x`` + - + +Network +~~~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil function + - Linux + - macOS + - BSD + - Windows + * - :func:`net_connections` + - ``netstat -anp``, ``ss``, ``lsof -nP -i -U`` + - ``netstat -anp`` + - ``netstat -an`` + - ``netstat -an`` + * - :func:`net_if_addrs` + - ``ifconfig``, ``ip addr`` + - ``ifconfig`` + - ``ifconfig`` + - ``ipconfig``, ``systeminfo`` + * - :func:`net_io_counters` + - ``netstat -i``, ``ifconfig``, ``ip -s link`` + - ``netstat -i`` + - ``netstat -i`` + - ``netstat -e`` + * - :func:`net_if_stats` + - ``ifconfig``, ``ip -br link``, ``ip link`` + - ``ifconfig`` + - ``ifconfig`` + - ``netsh interface show interface`` + +Sensors +~~~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil function + - Linux + - macOS + - BSD + - Windows + * - :func:`sensors_temperatures` + - ``sensors`` + - + - ``sysctl dev.cpu.*.temperature`` + - + * - :func:`sensors_fans` + - ``sensors`` + - + - ``sysctl dev.cpu.*.fan`` + - + * - :func:`sensors_battery` + - ``acpi -b`` + - ``pmset -g batt`` + - ``apm -b`` + - + +Other +~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil function + - Linux + - macOS + - BSD + - Windows + * - :func:`boot_time` + - ``uptime``, ``who -b`` + - ``sysctl kern.boottime`` + - ``sysctl kern.boottime`` + - ``systeminfo`` + * - :func:`users` + - ``who -a``, ``w`` + - same + - same + - + * - :func:`pids` + - ``ps -A -eo pid`` + - same + - same + - ``tasklist`` + * - :func:`pid_exists` + - ``kill -0 PID`` + - same + - same + - + +Process methods +--------------- + +Assuming ``p = psutil.Process()``. + +Identity +~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil method + - Linux + - macOS + - BSD + - Windows + * - :meth:`p.name() ` + - ``ps -o comm -p PID`` + - same + - ``procstat -b PID`` + - + * - :meth:`p.exe() ` + - ``readlink /proc/pid/exe`` + - ``lsof -p PID`` + - ``procstat -b PID`` + - + * - :meth:`p.cmdline() ` + - ``ps -o args -p PID`` + - same + - ``procstat -c PID`` + - + * - :meth:`p.status() ` + - ``ps -o stat -p PID`` + - same + - same + - + * - :meth:`p.create_time() ` + - ``ps -o lstart -p PID`` + - same + - same + - + * - :meth:`p.is_running() ` + - ``kill -0 PID`` + - same + - same + - + * - :meth:`p.environ() ` + - ``xargs -0 -a /proc/pid/environ`` + - + - ``procstat -e PID`` + - + * - :meth:`p.cwd() ` + - ``pwdx PID`` + - ``lsof -p PID -a -d cwd`` + - + - + +Process tree +~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil method + - Linux + - macOS + - BSD + - Windows + * - :meth:`p.ppid() ` + - ``ps -o ppid= -p PID`` + - same + - same + - + * - :meth:`p.parent() ` + - ``ps -p $(ps -o ppid= -p PID)`` + - same + - same + - + * - :meth:`p.parents() ` + - ``pstree -s PID`` + - same + - same + - + * - :meth:`p.children(recursive=False) ` + - ``pgrep -P PID`` + - same + - same + - + * - :meth:`p.children(recursive=True) ` + - ``pstree -p PID`` + - same + - same + - + +Credentials +~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil method + - Linux + - macOS + - BSD + - Windows + * - :meth:`p.uids() ` + - ``ps -o uid,ruid,suid -p PID`` + - same + - ``procstat -s PID`` + - + * - :meth:`p.gids() ` + - ``ps -o gid,rgid,sgid -p PID`` + - same + - ``procstat -s PID`` + - + * - :meth:`p.username() ` + - ``ps -o user -p PID`` + - same + - same + - + * - :meth:`p.terminal() ` + - ``ps -o tty -p PID`` + - same + - same + - + +CPU / scheduling +~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil method + - Linux + - macOS + - BSD + - Windows + * - :meth:`p.cpu_percent() ` + - ``ps -o %cpu -p PID`` + - same + - same + - + * - :meth:`p.cpu_times() ` + - ``ps -o cputime -p PID`` + - same + - ``procstat -r PID`` + - + * - :meth:`p.cpu_num() ` + - ``ps -o psr -p PID`` + - + - + - + * - :meth:`p.num_ctx_switches() ` + - ``pidstat -w -p PID`` + - + - ``procstat -r PID`` + - + * - :meth:`p.cpu_affinity() ` + - ``taskset -p PID`` + - + - ``cpuset -g -p PID`` + - + * - :meth:`p.cpu_affinity(CPUS) ` + - ``taskset -p MASK PID`` + - + - ``cpuset -s -p PID -l CPUS`` + - + * - :meth:`p.ionice() ` + - ``ionice -p PID`` + - + - + - + * - :meth:`p.ionice(CLASS) ` + - ``ionice -c CLASS -p PID`` + - + - + - + * - :meth:`p.nice() ` + - ``ps -o nice -p PID`` + - same + - same + - + * - :meth:`p.nice(VALUE) ` + - ``renice -n VALUE -p PID`` + - same + - same + - + * - :meth:`p.rlimit(RES) ` + - ``prlimit --pid PID`` + - + - ``procstat rlimit PID`` + - + * - :meth:`p.rlimit(RES, LIMITS) ` + - ``prlimit --pid PID --RES=SOFT:HARD`` + - + - + - + +Memory +~~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil method + - Linux + - macOS + - BSD + - Windows + * - :meth:`p.memory_info() ` + - ``ps -o rss,vsz -p PID`` + - same + - same + - + * - :meth:`p.memory_extras() ` + - ``cat /proc/pid/status`` + - + - + - + * - :meth:`p.memory_percent() ` + - ``ps -o %mem -p PID`` + - same + - same + - + * - :meth:`p.memory_maps() ` + - ``pmap PID`` + - ``vmmap PID`` + - ``procstat -v PID`` + - + * - :meth:`p.memory_footprint() ` + - ``smem``, ``smemstat`` + - + - + - + * - :meth:`p.page_faults() ` + - ``ps -o maj_flt,min_flt -p PID`` + - ``ps -o faults -p PID`` + - ``procstat -r PID`` + - + +Threads +~~~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil method + - Linux + - macOS + - BSD + - Windows + * - :meth:`p.num_threads() ` + - ``ps -o nlwp -p PID`` + - same + - same + - + * - :meth:`p.threads() ` + - ``ps -T -p PID`` + - + - + - + +Files and connections +~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil method + - Linux + - macOS + - BSD + - Windows + * - :meth:`p.net_connections() ` + - ``ss -p``, ``lsof -p PID -i`` + - ``lsof -p PID -i`` + - + - ``netstat -ano | findstr PID`` + * - :meth:`p.open_files() ` + - ``lsof -p PID`` + - same + - ``procstat -f PID``, ``fstat`` + - ``handle.exe -p PID`` + * - :meth:`p.io_counters() ` + - ``cat /proc/pid/io`` + - + - + - + * - :meth:`p.num_fds() ` + - ``ls /proc/pid/fd | wc -l`` + - + - + - + * - :meth:`p.num_handles() ` + - + - + - + - + +Signals +~~~~~~~ + +.. list-table:: + :header-rows: 1 + :class: wide-table + + * - psutil method + - Linux + - macOS + - BSD + - Windows + * - :meth:`p.send_signal() ` + - ``kill -SIG PID`` + - same + - same + - + * - :meth:`p.suspend() ` + - ``kill -STOP PID`` + - same + - same + - + * - :meth:`p.resume() ` + - ``kill -CONT PID`` + - same + - same + - + * - :meth:`p.terminate() ` + - ``kill -TERM PID`` + - same + - same + - ``taskkill /PID PID`` + * - :meth:`p.kill() ` + - ``kill -KILL PID`` + - same + - same + - ``taskkill /F /PID PID`` + * - :meth:`p.wait() ` + - ``tail --pid=PID -f /dev/null`` + - ``lsof -p PID +r 1`` + - ``pwait PID`` + - diff --git a/docs/stdlib-equivalents.rst b/docs/stdlib-equivalents.rst new file mode 100644 index 0000000000..24cc9ac3a7 --- /dev/null +++ b/docs/stdlib-equivalents.rst @@ -0,0 +1,289 @@ +Stdlib equivalents +================== + +This page maps psutil's Python API to the closest equivalent in the Python +standard library. This is useful for understanding what psutil replaces and how +the two APIs differ. The most common difference is that stdlib functions only +operate on the **current process**, while psutil works on **any process** +(PID). + +.. seealso:: + - :doc:`shell-equivalents` + - :doc:`alternatives` + +System-wide functions +--------------------- + +CPU +~~~ + +.. list-table:: + :class: longtable wide-table + :header-rows: 1 + :widths: 30 60 100 + + * - psutil + - stdlib + - notes + * - :func:`cpu_count` + - :func:`os.cpu_count`, + :func:`multiprocessing.cpu_count` + - Same as ``cpu_count(logical=True)``; no support for + physical cores (``logical=False``). See :ref:`FAQ `. + * - :func:`cpu_count` + - :func:`os.process_cpu_count` + - CPUs the process is allowed to use (Python 3.13+). + Equivalent to ``len(psutil.Process().cpu_affinity())``. + See :ref:`FAQ `. + * - :func:`getloadavg` + - :func:`os.getloadavg` + - Same on POSIX; psutil also supports Windows. + +Disk +~~~~ + +.. list-table:: + :class: longtable wide-table + :header-rows: 1 + :widths: 45 45 100 + + * - psutil + - stdlib + - notes + * - :func:`disk_usage` + - :func:`shutil.disk_usage` + - Same as :func:`shutil.disk_usage`; psutil also adds + :field:`percent`. Added to CPython 3.3 (:bpo:`12442`). + * - :func:`disk_partitions` + - :func:`os.listdrives`, + :func:`os.listmounts`, + :func:`os.listvolumes` + - Windows only (Python 3.12+). Low-level APIs: + drive letters, volume GUIDs, mount points. psutil + combines them in one cross-platform call. + +Network +~~~~~~~ + +.. list-table:: + :class: longtable wide-table + :header-rows: 1 + :widths: 45 45 100 + + * - psutil + - stdlib + - notes + * - :func:`net_if_addrs` + - :func:`socket.if_nameindex` + - Stdlib returns :term:`NIC` names only; psutil also returns + addresses, netmasks, broadcast, and PTP. + +Process +~~~~~~~ + +.. list-table:: + :class: longtable wide-table + :header-rows: 1 + :widths: 45 45 100 + + * - psutil + - stdlib + - notes + * - :func:`pid_exists` + - ``os.kill(pid, 0)`` + - Common POSIX idiom; psutil also supports Windows. + +Process methods +--------------- + +Assuming ``p = psutil.Process()``. + +Identity +~~~~~~~~ + +.. list-table:: + :class: longtable wide-table + :header-rows: 1 + :widths: 45 45 100 + + * - psutil + - stdlib + - notes + * - :meth:`p.pid ` + + - :func:`os.getpid` + - + * - :meth:`p.ppid() ` + - :func:`os.getppid` + - + * - :meth:`p.cwd() ` + - :func:`os.getcwd` + - + * - :meth:`p.environ() ` + - :data:`os.environ` + - Will differ from launch environment if modified at runtime. + * - :meth:`p.exe() ` + - :data:`sys.executable` + - Python interpreter path only. + * - :meth:`p.cmdline() ` + - :data:`sys.argv` + - Python process only. + +Credentials +~~~~~~~~~~~ + +.. list-table:: + :class: longtable wide-table + :header-rows: 1 + :widths: 30 65 50 + + * - psutil + - stdlib + - notes + * - :meth:`p.uids() ` + - :func:`os.getuid`, + :func:`os.geteuid`, + :func:`os.getresuid` + - + * - :meth:`p.gids() ` + - :func:`os.getgid`, + :func:`os.getegid`, + :func:`os.getresgid` + - + * - :meth:`p.username() ` + - :func:`os.getlogin`, :func:`getpass.getuser` + - Rough equivalent; not per-process. + +CPU / scheduling +~~~~~~~~~~~~~~~~ + +.. list-table:: + :class: longtable wide-table + :header-rows: 1 + :widths: 60 60 100 + + * - psutil + - stdlib + - notes + * - :meth:`p.cpu_times() ` + - :func:`os.times` + - :func:`os.times` also has ``elapsed``; psutil adds + :field:`iowait` (Linux). + * - :meth:`p.cpu_times() ` + - :func:`resource.getrusage` + - ``ru_utime`` / ``ru_stime`` match; have higher precision. + * - :meth:`p.num_ctx_switches() ` + - :func:`resource.getrusage` + - Current process only; psutil works for any PID. + * - :meth:`p.nice() ` + - :func:`os.getpriority`, + :func:`os.setpriority` + - POSIX only; psutil also supports Windows. + Added to CPython 3.3 (:bpo:`10784`). + * - :meth:`p.nice() ` + - :func:`os.nice` + - POSIX only; psutil also supports Windows. + * - *no equivalent* + - :func:`os.sched_getscheduler`, + :func:`os.sched_setscheduler` + - Sets scheduling *policy* (``SCHED_*``). Unlike nice, + which sets priority within ``SCHED_OTHER``. Real-time + policies preempt normal processes. + * - :meth:`p.cpu_affinity() ` + - :func:`os.sched_getaffinity`, + :func:`os.sched_setaffinity` + - Nearly equivalent; both accept a PID. Stdlib is + Linux/BSD; psutil also supports Windows. + * - :meth:`p.rlimit() ` + - :func:`resource.getrlimit`, + :func:`resource.setrlimit` + - Same interface; psutil works for any PID (Linux only). + +Memory +~~~~~~ + +.. list-table:: + :class: longtable wide-table + :header-rows: 1 + :widths: 45 45 100 + + * - psutil + - stdlib + - notes + * - :meth:`p.memory_info() ` + - :func:`resource.getrusage` + - Only :term:`peak_rss` (``ru_maxrss``); psutil returns :term:`RSS`, :term:`VMS`, and more. + * - :meth:`p.page_faults() ` + - :func:`resource.getrusage` + - Current process only. + +I/O +~~~ + +.. list-table:: + :class: longtable wide-table + :header-rows: 1 + :widths: 45 45 100 + + * - psutil + - stdlib + - notes + * - :meth:`p.io_counters() ` + - :func:`resource.getrusage` + - Block I/O only (``ru_inblock`` / ``ru_oublock``), + current process. psutil returns bytes and counts for any PID. + +Threads +~~~~~~~ + +.. list-table:: + :class: longtable wide-table + :header-rows: 1 + :widths: 50 55 100 + + * - psutil + - stdlib + - notes + * - :meth:`p.num_threads() ` + - :func:`threading.active_count` + - Stdlib counts Python threads only; psutil counts all OS threads. + * - :meth:`p.threads() ` + - :func:`threading.enumerate` + - Stdlib returns :class:`threading.Thread` objects; psutil + returns OS thread IDs with CPU times. + +Signals +~~~~~~~ + +.. list-table:: + :class: longtable wide-table + :header-rows: 1 + :widths: 45 55 100 + + * - psutil + - stdlib + - notes + * - :meth:`p.send_signal() ` + - :func:`os.kill` + - Same on POSIX; limited on Windows. psutil adds + :exc:`NoSuchProcess` / :exc:`AccessDenied` and avoids + killing reused PIDs. + * - :meth:`p.suspend() ` + - :func:`os.kill` + :data:`signal.SIGSTOP` + - Same as above. + * - :meth:`p.resume() ` + - :func:`os.kill` + :data:`signal.SIGCONT` + - Same as above. + * - :meth:`p.terminate() ` + - :func:`os.kill` + :data:`signal.SIGTERM` + - Same as above. On Windows uses ``TerminateProcess()``. + * - :meth:`p.kill() ` + - :func:`os.kill` + :data:`signal.SIGKILL` + - Same as above. On Windows uses ``TerminateProcess()``. + * - :meth:`p.wait() ` + - :func:`os.waitpid` + - Child processes only; psutil works for any PID. + * - :meth:`p.wait() ` + - :meth:`subprocess.Popen.wait` + - Equivalent; psutil uses efficient OS-level waiting on + Linux/BSD. Added to CPython 3.15 (:cpy-pr:`144047`). diff --git a/docs/test_docs.py b/docs/test_docs.py new file mode 100644 index 0000000000..0eaea664ed --- /dev/null +++ b/docs/test_docs.py @@ -0,0 +1,1040 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Sanity checks for the Sphinx docs and blog posts.""" + +import importlib.util +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +import zlib +from datetime import datetime +from datetime import timezone + +import pytest + +HERE = pathlib.Path(__file__).resolve().parent +ROOT = HERE.parent +DOCS = ROOT / "docs" +BLOG = DOCS / "blog" + +VALID_BLOG_TAGS = frozenset({ + # platforms + "bsd", + "linux", + "macos", + "sunos", + "windows", + # API / compatibility + "api-design", + "compatibility", + "new-api", + "new-platform", + # editorial + "featured", + # topics + "c", + "community", + "memory", + "performance", + "personal", + "python-core", + "release", + "tests", + "wheels", +}) + +sys.path.insert(0, str(HERE)) # so that "import conf" wins +import conf # noqa: E402 +import label_role # noqa: E402 +import substitutions # noqa: E402 +from testutil import feed_urls # noqa: E402 +from testutil import find_canonical # noqa: E402 +from testutil import og_value # noqa: E402 + +pytestmark = pytest.mark.skipif( + sys.platform == "win32", reason="docs are built on Linux only" +) + +HTML_DIR = None + +PAST_RELEASE_DIRS = frozenset( + entry["url"].strip("/") + for entry in conf.VERSIONS["versions"] + if entry.get("ref") +) + + +def _under(path): + return HTML_DIR in path.parents or path == HTML_DIR + + +@pytest.fixture(scope="module") +def build_html(): + """Build the site once. `make html` already turns warnings into + errors by default. + """ + global HTML_DIR + + tmp = tempfile.mkdtemp() + subprocess.check_call([ + "make", + "-C", + str(DOCS), + "html", + f"BUILDDIR={tmp}", + f"PYTHON={sys.executable}", + ]) + HTML_DIR = pathlib.Path(tmp) / "html" + yield + shutil.rmtree(tmp) + HTML_DIR = None + + +def blog_posts(): + return sorted(BLOG.rglob("*.rst")) + + +def read_html(*parts): + # dirhtml writes each page as /index.html; only the root + # index.html stays flat. Translate the historical ".html". + rel = pathlib.Path(*parts) + if rel.name == "index.html": + path = HTML_DIR / rel + else: + path = HTML_DIR / rel.with_suffix("") / "index.html" + return path.read_text() + + +def post_title(rst): + """First H1 in a .rst file (the line above a `===` underline).""" + lines = rst.read_text().splitlines() + for i in range(1, len(lines)): + if re.fullmatch(r"=+", lines[i].strip()): + return lines[i - 1].strip() + raise ValueError(f"no RST H1 found in {rst}") + + +def post_tags(rst): + """Tags from a blog post's `:tags:` line, or [] if absent.""" + m = re.search(r":tags:\s*(.+)", rst.read_text()) + if not m: + return [] + return [t.strip() for t in m.group(1).split(",")] + + +def blog_html(rst): + """Read the built HTML for a blog post given its source .rst.""" + rel = rst.relative_to(BLOG).with_suffix("") + return (HTML_DIR / "blog" / rel / "index.html").read_text() + + +def source_rst_for(html_path): + """Return the .rst source for a built page, or None for pages + generated without a source (ablog archives, sphinx auto-pages + like genindex/search/py-modindex). + """ + rel = html_path.relative_to(HTML_DIR) + if rel.name != "index.html": + return None + if rel.parent == pathlib.Path("."): + slug = "index" # root home page + else: + slug = rel.parent.as_posix() + src = DOCS / (slug + ".rst") + return src if src.is_file() else None + + +def all_html_pages(): + """Every "real content" HTML page in the build.""" + for p in sorted(HTML_DIR.rglob("*.html")): + rel_parts = p.relative_to(HTML_DIR).parts + if any(part.startswith("_") for part in rel_parts): + continue + if source_rst_for(p) is None: + continue + yield p + + +class TestSourceRefs: + """Checks on the .rst sources, no build needed.""" + + def test_src_role_targets_exist(self, subtests): + # :src:`path` links to the file on GitHub. Nothing validates + # the path: rename or move the file and the link 404s, with + # no build warning. Both the bare and the labelled + # (`text `) forms are used. + pat = re.compile(r":src:`([^`]+)`") + for rst in sorted(DOCS.rglob("*.rst")): + for target in pat.findall(rst.read_text()): + m = re.search(r"<([^>]+)>", target) + if m: + target = m.group(1) + with subtests.test(rst=rst.relative_to(ROOT), ref=target): + assert (ROOT / target).exists() + + def test_first_commit_date(self): + # _ext/substitutions.py hardcodes it to keep git out of the + # build. Check it against the real history. + cmd = ["git", "log", "--reverse", "--format=%ct"] + out = subprocess.check_output(cmd, cwd=ROOT).split(b"\n", 1)[0] + first = datetime.fromtimestamp(int(out), tz=timezone.utc) + assert first.date() == substitutions.FIRST_COMMIT + + +class TestBlogPostFiles: + + def test_every_file_has_directives(self): + missing = [] + for p in blog_posts(): + text = p.read_text() + missed = [ + k for k in (".. post::", ":author:", ":tags:") if k not in text + ] + if missed: + rel = p.relative_to(ROOT) + missing.append(f"{rel}: missing {missed}") + assert missing == [] + + def test_date_year_matches_path_year(self): + mismatches = [] + for p in blog_posts(): + m = re.search(r"\.\. post:: (\d{4})-", p.read_text()) + if m and m.group(1) != p.parent.name: + rel = p.relative_to(ROOT) + mismatches.append( + f"{rel}: date-year={m.group(1)} dir={p.parent.name}" + ) + assert mismatches == [] + + def test_valid_tags(self): + invalid = {} + for p in blog_posts(): + bad = [t for t in post_tags(p) if t not in VALID_BLOG_TAGS] + if bad: + invalid[str(p.relative_to(ROOT))] = bad + assert not invalid, f"invalid tags: {invalid}" + + +@pytest.mark.usefixtures("build_html") +class TestHtmlBuild: + """Build sanity: pages exist, extensions wire up, footer dates.""" + + def test_files_exist(self): + assert (HTML_DIR / "index.html").exists() + assert (HTML_DIR / "blog" / "index.html").exists() + + def test_page_sources_are_published(self, subtests): + for name in ("api", "faq", "install", "blog/2025/drop-py27"): + with subtests.test(page=name): + src = HTML_DIR / "_sources" / (name + ".rst.txt") + assert src.is_file() is True + + def test_github_pages_control_files(self): + # sphinx.ext.githubpages writes these: .nojekyll stops Jekyll + # from dropping _static/, CNAME is derived from html_baseurl. + assert (HTML_DIR / ".nojekyll").is_file() + assert (HTML_DIR / "CNAME").read_text().strip() == "psutil.io" + + def test_substitutions_expanded(self): + # _ext/substitutions.py expands {{years_in_development}}. If + # the hook breaks, the literal token ships instead. + html = read_html("index.html") + assert f"{substitutions.years_in_development()} years" in html + assert "{{" not in html + + def test_changelog_anchors(self): + # Indirectly test _ext/changelog_anchors.py. Every X.Y.Z + # version heading in changelog.rst must get an `id="XYZ"` + # anchor in the rendered HTML. + source = (DOCS / "changelog.rst").read_text() + html = read_html("changelog.html") + missing = [] + for m in re.finditer(r"^(\d+)\.(\d+)\.(\d+)", source, re.MULTILINE): + anchor = "".join(m.groups()) + if f'id="{anchor}"' not in html: + missing.append(m.group(0)) + assert missing == [] + + def test_gh_role_resolves(self): + # Checks extlinks = {"gh": ...} in conf.py. Without it, every + # :gh:`NNN` in the docs silently renders as plain text instead + # of links. + html = read_html("changelog.html") + assert 'href="https://github.com/giampaolo/psutil/issues/' in html + + def test_label_role_resolves(self): + source = (DOCS / "changelog.rst").read_text() + html = read_html("changelog.html") + for label in label_role.LABELS: + badge = f'class="cl-label cl-label-{label}">{label}<' + assert source.count(f":label:`{label}`") == html.count(badge) + + def test_intersphinx_resolves(self): + # Check intersphinx_mapping["python"] in conf.py. If it does + # not work, :mod:`func` refs to the stdlib silently render as + # plain text instead of links. + html = read_html("api.html") + assert 'href="https://docs.python.org/3/' in html + + def test_footer_last_updated_matches_git(self, subtests): + def is_merge_checkout(): + ret = subprocess.run( + ["git", "rev-parse", "-q", "--verify", "HEAD^2"], + capture_output=True, + check=False, + ) + return ret.returncode == 0 + + if is_merge_checkout(): + pytest.skip("merge commit") + # Skip pages that pull in external files via `.. include::` + # or `.. raw:: :file:`. sphinx-last-updated-by-git + # walks those deps and picks the latest commit timestamp, + # which we don't replicate here. + dep_pat = re.compile(r"^\.\. (?:include|raw)::", re.MULTILINE) + date_pat = re.compile(r"Updated: ]*>(\d{4}-\d{2}-\d{2})") + for html in all_html_pages(): + src = source_rst_for(html) + if dep_pat.search(src.read_text()): + continue + m = date_pat.search(html.read_text()) + if m is None: + continue # page doesn't render the footer date + cmd = [ + "git", + "log", + "-1", + "--author-date-order", + "--format=%at", + "--", + str(src), + ] + ts = subprocess.check_output(cmd).strip() + if not ts: + continue # untracked source file + expected = datetime.fromtimestamp( + int(ts), tz=timezone.utc + ).strftime("%Y-%m-%d") + with subtests.test(page=html.relative_to(HTML_DIR)): + assert m.group(1) == expected + + def test_footer_github_links_resolve(self, subtests): + # ablog and codeautolink generate pages with no .rst behind + # them (blog/tag/*, blog/2025, _modules/*). Their footer + # links used to be built from the page name anyway, so they + # 404ed on GitHub. + pat = re.compile( + r'github\.com/giampaolo/psutil/(?:edit|commits)/master/(\S+?)"' + ) + for page in sorted(HTML_DIR.rglob("*.html")): + html = page.read_text(encoding="utf-8", errors="replace") + for target in set(pat.findall(html)): + with subtests.test(page=page.relative_to(HTML_DIR)): + assert (ROOT / target).is_file() + + +@pytest.mark.usefixtures("build_html") +class TestSitemap: + + def test_known_pages_listed(self, subtests): + # sphinx-sitemap should emit one per built HTML page + # (source docs + blog posts + ablog-generated pages), rooted at + # html_baseurl. dirhtml gives directory-style URLs (trailing + # slash); the home page is the bare root. + sitemap = HTML_DIR / "sitemap.xml" + assert sitemap.exists() + ns = {"s": "http://www.sitemaps.org/schemas/sitemap/0.9"} + tree = ET.parse(sitemap) + urls = {u.text for u in tree.getroot().findall("s:url/s:loc", ns)} + for page in ( + "", + "api/", + "changelog/", + "blog/2026/event-driven-process-waiting/", + ): + with subtests.test(page=page): + assert conf.html_baseurl + page in urls + + def test_no_duplicate_urls(self): + # ablog re-renders the blog index on top of blog.rst, so + # sphinx-sitemap lists that URL twice unless we dedupe. + sitemap = HTML_DIR / "sitemap.xml" + ns = {"s": "http://www.sitemaps.org/schemas/sitemap/0.9"} + urls = [ + u.text + for u in ET.parse(sitemap).getroot().findall("s:url/s:loc", ns) + ] + dupes = sorted({u for u in urls if urls.count(u) > 1}) + assert dupes == [] + + def test_listed_pages_have_description(self, subtests): + # ablog's generated pages come with no doctree, so + # sphinxext-opengraph skips them and emits no description. + pat = re.compile(r']*name="description"', re.IGNORECASE) + sitemap = HTML_DIR / "sitemap.xml" + ns = {"s": "http://www.sitemaps.org/schemas/sitemap/0.9"} + for url in ET.parse(sitemap).getroot().findall("s:url/s:loc", ns): + rel = url.text.replace(conf.html_baseurl, "") + page = HTML_DIR / rel / "index.html" + with subtests.test(page=rel or "/"): + assert pat.search(page.read_text(errors="replace")) + + def test_excludes_utility_pages(self, subtests): + # sitemap_excludes must use the dirhtml dir form ("search/", + # not "search.html") or utility + ablog pages leak in. + sitemap = HTML_DIR / "sitemap.xml" + ns = {"s": "http://www.sitemaps.org/schemas/sitemap/0.9"} + urls = { + u.text + for u in ET.parse(sitemap).getroot().findall("s:url/s:loc", ns) + } + for slug in ( + "genindex", + "py-modindex", + "search", + "404", + "blog/archive", + "blog/drafts", + "blog/2026", + ): + with subtests.test(slug=slug): + assert conf.html_baseurl + slug + "/" not in urls + + def test_has_every_blog_post(self, subtests): + # Catches drift between ablog's post registry and + # sphinx-sitemap. If sphinx-sitemap stops listing some posts + # (e.g. due to an ablog version bump), search engines stop + # discovering them. + sitemap = HTML_DIR / "sitemap.xml" + ns = {"s": "http://www.sitemaps.org/schemas/sitemap/0.9"} + urls = { + u.text + for u in ET.parse(sitemap).getroot().findall("s:url/s:loc", ns) + } + for rst in blog_posts(): + rel = rst.relative_to(BLOG).with_suffix("") + expected = conf.html_baseurl + "blog/" + rel.as_posix() + "/" + with subtests.test(rst=rst): + assert expected in urls + + +@pytest.mark.usefixtures("build_html") +class TestCanonicalUrl: + + def test_link_on_pages(self, subtests): + # Sphinx emits using html_baseurl. If + # html_baseurl is misconfigured, every shared URL points at a + # 404. + for page in ( + "index.html", + "api.html", + "blog/2026/event-driven-process-waiting.html", + ): + url = find_canonical(read_html(page)) + with subtests.test(page=page): + assert url is not None + assert url.startswith(conf.html_baseurl) + + def test_og_urls_rooted_at_baseurl(self, subtests): + # og:url + og:image must point at the deployed domain; they + # once lagged html_baseurl and pointed at the old host. + html = read_html("api.html") + for prop in ("og:url", "og:image"): + val = og_value(html, prop) + with subtests.test(prop=prop): + assert val is not None + assert val.startswith(conf.html_baseurl) + + +@pytest.mark.usefixtures("build_html") +class TestRightToc: + + def test_visibility(self, subtests): + has_toc = ("api.html", "faq.html", "glossary.html", "blog.html") + no_toc = ("index.html", "genindex.html", "search.html") + pat = re.compile(r']*\bclass="[^"]*\bright-sidebar\b') + for file in has_toc: + with subtests.test(file=file): + assert pat.search(read_html(file)) + for file in no_toc: + with subtests.test(file=file): + assert not pat.search(read_html(file)) + + def test_hash_targets_resolve(self, subtests): + # Every inside the right-sidebar must point to + # an id that exists on the same page. Otherwise the JS + # hash-match path in right-toc.js silently misses on direct + # URL load. + html = read_html("api.html") + m = re.search( + r']*\bclass="[^"]*\bright-sidebar\b[^"]*"[^>]*>(.*?)', + html, + re.DOTALL, + ) + assert m + hrefs = re.findall(r'href="#([^"]+)"', m.group(1)) + assert hrefs + ids_on_page = set(re.findall(r'\bid="([^"]+)"', html)) + for href in hrefs: + with subtests.test(href=href): + assert href in ids_on_page + + def test_glossary_mode(self): + # Tests docs/_ext/glossary_toc.py, which injects + # glossary_terms into the Jinja context. + html = read_html("glossary.html") + assert 'data-toc-mode="glossary"' in html + m = re.search( + r'', + html, + re.DOTALL, + ) + assert m, "right-sidebar--glossary aside not found" + terms = re.findall(r'([^<]+)', m.group(1)) + assert terms + assert terms == sorted(terms, key=str.lower) + + +@pytest.mark.usefixtures("build_html") +class TestCodeAutoLink: + """Checks sphinx-codeautolink integration.""" + + def test_enabled(self): + html = read_html("api-overview.html") + assert "sphinx-codeautolink-a" in html + + def test_resolves_process_instance_methods(self): + # Check that things like `p.name()`are resolved. + html = read_html("api-overview.html") + assert ( + ' must appear only + # once. layout.html used to inject it unconditionally, + # duplicating ablog's auto-injection. + needle = 'type="application/atom+xml"' + for html in all_html_pages(): + with subtests.test(page=html.relative_to(HTML_DIR)): + head = html.read_text()[:8192] + assert head.count(needle) == 1 + + +@pytest.mark.usefixtures("build_html") +class TestOpenGraph: + """og:* and twitter:* social-preview meta tags.""" + + def test_tags_on_source_pages(self, subtests): + for rst in sorted(DOCS.rglob("*.rst")): + if rst.name == "blog.rst": + continue + rel = rst.relative_to(DOCS) + if rel.name == "index.rst" and rel.parent == pathlib.Path("."): + html_path = HTML_DIR / "index.html" + else: + html_path = HTML_DIR / rel.with_suffix("") / "index.html" + html = html_path.read_text() + with subtests.test(rst=rst): + for needle in ( + "og:title", + "og:site_name", + "og:description", + "og:image", + ): + assert needle in html + + def test_tags_on_blog_posts(self, subtests): + # sphinxext-opengraph should emit og:* meta tags on every + # post page so that shared URLs render as rich previews. + properties = ( + "og:title", + "og:url", + "og:site_name", + "og:description", + "og:image", + ) + for rst in blog_posts(): + html = blog_html(rst) + for prop in properties: + with subtests.test(rst=rst, prop=prop): + assert f'property="{prop}"' in html + + def test_type(self, subtests): + # For blog posts it must be og:type="article" else "website". + for rst in blog_posts(): + with subtests.test(rst=rst, expected="article"): + assert og_value(blog_html(rst), "og:type") == "article" + for page in ("index.html", "api.html", "install.html"): + with subtests.test(page=page, expected="website"): + assert og_value(read_html(page), "og:type") == "website" + + def test_blog_index_metadata(self, subtests): + # Regression: ablog renders blog.html without a doctree, so + # sphinxext-opengraph skips emitting og:* tags. Our + # opengraph_override extension synthesizes them. + html = read_html("blog.html") + for tag in ( + 'name="description"', + 'property="og:title"', + 'property="og:type"', + 'property="og:url"', + 'property="og:site_name"', + 'property="og:description"', + 'property="og:image"', + 'name="twitter:card"', + ): + with subtests.test(tag=tag): + assert tag in html + + def test_description_uses_post_excerpt(self): + # Blog post og:description (and the social card PNG sourced + # from the same value) should be the curated excerpt from the + # `.. post::` directive body, not the page body's first + # paragraph. Spot-check on a known post. + html = read_html("blog", "2026", "event-driven-process-waiting.html") + desc = og_value(html, "og:description") + # Excerpt opens with "Replacing"; the page body opens with + # "One of the less fun aspects of process management ...". + assert desc is not None + assert desc.startswith("Replacing") + + def test_description_excludes_post_banner(self, subtests): + # Regression: the post banner (author/date/readtime/tags) + # used to leak into og:description, making social previews + # start with "Giampaolo Rodola 2026-01-28 5 min read ...". + for rst in blog_posts(): + desc = og_value(blog_html(rst), "og:description") + with subtests.test(rst=rst): + assert desc is not None + assert not desc.startswith("Giampaolo") + assert "min read" not in desc[:80] + + +@pytest.mark.usefixtures("build_html") +class TestBlogPosts: + """Rendered blog post pages and the blog index listing.""" + + def test_banner_on_every_post(self, subtests): + css_names = ( + "post-meta-banner", + "post-meta-author", + "post-meta-date", + "post-meta-readtime", + "post-meta-tags", + ) + for rst in blog_posts(): + html = blog_html(rst) + for css in css_names: + with subtests.test(rst=rst, css=css): + assert css in html + + def test_listing_has_all_posts(self, subtests): + # blog.html should list all posts, each containing the post's + # title. + posts = blog_posts() + html = read_html("blog.html") + assert len(re.findall(r'
  • (.*?)
  • ', html, re.DOTALL + ) + posts = blog_posts() + assert len(summaries) == len(posts) + for i, summary in enumerate(summaries): + plain = re.sub(r"<[^>]+>", "", summary).strip() + with subtests.test(card=i): + assert plain + + def test_featured_pill(self, subtests): + # Posts with the "featured" tag show a "Featured" pill in + # their post-meta banner via _ext/post_banner.py. Other posts + # must not. + for rst in blog_posts(): + is_featured = "featured" in post_tags(rst) + html = blog_html(rst) + with subtests.test(rst=rst, is_featured=is_featured): + if is_featured: + assert "post-meta-featured" in html + else: + assert "post-meta-featured" not in html + + +@pytest.mark.usefixtures("build_html") +class TestNoExternalAssets: + """Stylesheets and fonts are self-hosted, not pulled from a CDN.""" + + ASSET_LINK_RE = re.compile( + r']*\brel="(?:stylesheet|preload)"[^>]*>', re.IGNORECASE + ) + HREF_RE = re.compile(r'\bhref="([^"]+)"') + + @staticmethod + def is_external(url): + return url.startswith(("http://", "https://", "//")) + + def test_no_external_stylesheets(self, subtests): + # Analytics ' + "" + f'" + ) + + +def inject(html_dir, entry, current): + count = 0 + for path in sorted(html_dir.rglob("*.html")): + text = path.read_text(encoding="utf-8") + if "" not in text: + sys.stderr.write(f"warning: no in {path}\n") + continue + depth = len(path.relative_to(html_dir).parts) + root = "../" * depth + snippet = banner(entry, current, root) + path.write_text( + text.replace("", snippet + "", 1), encoding="utf-8" + ) + count += 1 + return count + + +def trim(html_dir): + for rel in DROP: + target = html_dir / rel + if target.is_dir(): + shutil.rmtree(target) + for rel in FONT_DIRS: + font_dir = html_dir / rel + if not font_dir.is_dir(): + continue + for pattern in LEGACY_FONTS: + for path in font_dir.rglob(pattern): + path.unlink() + + +def build_one(entry, current, site_dir): + tmp = pathlib.Path(tempfile.mkdtemp(prefix="psutil-docs-")) + worktree = tmp / "src" + try: + run( + ["git", "worktree", "add", "--detach", worktree, entry["ref"]], + cwd=ROOT, + ) + env = tmp / "venv" + venv.create(env, with_pip=True) + python = env / "bin" / "python" + run([ + python, + "-m", + "pip", + "install", + "--quiet", + "-r", + worktree / "docs" / "requirements.txt", + ]) + run(["make", "html", f"PYTHON={python}"], cwd=worktree / "docs") + html_dir = worktree / "docs" / "_build" / "html" + pages = inject(html_dir, entry, current) + trim(html_dir) + dst = site_dir / entry["url"].strip("/") + if dst.exists(): + shutil.rmtree(dst) + shutil.copytree(html_dir, dst) + size = sum(p.stat().st_size for p in dst.rglob("*") if p.is_file()) + print(f" {entry['name']}: {pages} pages, {size / 1048576:.1f} MB") + finally: + subprocess.call( + ["git", "worktree", "remove", "--force", worktree], cwd=ROOT + ) + shutil.rmtree(tmp, ignore_errors=True) + + +def parse_cli(): + global SITE_DIR, ONLY + parser = argparse.ArgumentParser( + description="Build past doc releases into a built site." + ) + parser.add_argument("site", help="built HTML dir, e.g. docs/_build/html") + parser.add_argument( + "--only", default=None, help="build just this version name" + ) + args = parser.parse_args() + SITE_DIR = pathlib.Path(args.site).resolve() + ONLY = args.only + + +def main(): + parse_cli() + if not SITE_DIR.is_dir(): + sys.exit(f"error: {SITE_DIR} does not exist; run `make html` first") + current, entries = load_versions() + if ONLY: + entries = [e for e in entries if e["name"] == ONLY] + if not entries: + sys.exit(f"error: no version named {ONLY!r} with a ref") + if not entries: + print("no past versions to build") + return + for entry in entries: + print(f"building {entry['name']} from {entry['ref']}") + build_one(entry, current, SITE_DIR) + + +if __name__ == "__main__": + main() diff --git a/scripts/internal/docs/find_adopters.py b/scripts/internal/docs/find_adopters.py new file mode 100755 index 0000000000..4c28c9c381 --- /dev/null +++ b/scripts/internal/docs/find_adopters.py @@ -0,0 +1,799 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +r"""Search GitHub for notable projects that use a given project +as a dependency. + +How it works: + +1. Enumerate all popular Python repos on GitHub via GraphQL + (paginated, >=MIN_STARS). +2. Batch-fetch data needed for the requested filters + (README content, dependency files). +3. Apply filters. Only repos passing ALL specified filters + are confirmed. Available filters: + - --inreadme : PROJECT mentioned in the README + - --indeps : PROJECT mentioned in dep files + (pyproject.toml, setup.py, setup.cfg, requirements*.txt) + +At least one filter must be specified. + +Output is RsT formatted, ready to paste into docs/adoption.rst. + +Usage: + python3 scripts/internal/docs/find_adopters.py \ + --project=psutil \ + --token=~/.github.api.key \ + --skip-file-urls=docs/adoption.rst \ + --min-stars=10000 --indeps +""" + +import argparse +import os +import pickle +import re +import sys +import time + +import requests + +from psutil._common import hilite +from psutil._common import print_color + +GITHUB_GRAPHQL = "https://api.github.com/graphql" +_CACHE_FILE = ".find_adopters.cache" + +# Set by parse_cli(). +PROJECT = "" +MIN_STARS = 0 +MAX_STARS = 0 +TOKEN = "" +SKIP = set() +INREADME = False +INDEPS = False +NO_CACHE = False + +# Fixed files to check for dependency declarations. +_FIXED_DEP_FILES = { + "pyproject.toml": "pyprojectToml", + "setup.py": "setupPy", + "setup.cfg": "setupCfg", + "requirements.txt": "requirementsTxt", +} + +# Max repos to batch in a single GraphQL query. +_BATCH_SIZE = 5 + + +green = lambda msg: hilite(msg, color="green") # noqa: E731 +yellow = lambda msg: hilite(msg, color="yellow") # noqa: E731 + + +def stderr(msg="", color=None): + if color: + print_color(msg, color=color, file=sys.stderr) + else: + print(msg, file=sys.stderr) + + +def graphql(session, query, variables=None): + """Execute a GraphQL query. Returns the 'data' dict.""" + payload = {"query": query} + if variables: + payload["variables"] = variables + try: + resp = session.post(GITHUB_GRAPHQL, json=payload) + except requests.exceptions.RequestException as err: + stderr(f" GraphQL request error: {err}") + return None + if resp.status_code != 200: + stderr(f" GraphQL HTTP error: {resp.status_code} {resp.text}") + return None + body = resp.json() + if "errors" in body: + for err in body["errors"]: + stderr(f" GraphQL error: {err.get('message', err)}") + return None + return body.get("data") + + +def get_session(token): + s = requests.Session() + s.headers["Authorization"] = f"Bearer {token}" + s.headers["Content-Type"] = "application/json" + return s + + +# --- Enumerate repos --- + + +def enumerate_repos(session): + """Enumerate all Python repos with >=MIN_STARS on GitHub.""" + stars_q = f"stars:>={MIN_STARS}" + if MAX_STARS: + stars_q = f"stars:{MIN_STARS}..{MAX_STARS}" + search_q = f"language:Python {stars_q}" + query = """ + query($q: String!, $first: Int!, $after: String) { + search(query: $q, type: REPOSITORY, + first: $first, after: $after) { + repositoryCount + edges { + node { + ... on Repository { + nameWithOwner + owner { login } + name + url + description + stargazerCount + isArchived + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ + results = [] + cursor = None + page = 0 + while True: + page += 1 + variables = { + "q": search_q, + "first": 100, + "after": cursor, + } + data = graphql(session, query, variables) + if data is None: + break + search_data = data["search"] + edges = search_data["edges"] + if not edges: + break + for edge in edges: + node = edge["node"] + if not node: + continue + results.append({ + "full_name": node["nameWithOwner"], + "owner": node["owner"]["login"], + "repo": node["name"], + "stars": node["stargazerCount"], + "description": node.get("description") or "", + "html_url": node["url"], + "archived": node["isArchived"], + }) + stderr( + f" page {page}: got {len(edges)} repos " + f"(total so far: {len(results)})" + ) + page_info = search_data["pageInfo"] + if not page_info["hasNextPage"]: + break + cursor = page_info["endCursor"] + time.sleep(1) + return results + + +# --- Fetch README --- + + +def fetch_readmes(session, repos): + """Batch-fetch README content for a list of repos. + + Returns a dict mapping full_name to README text + (or empty string if not found). + """ + result = {} + total = len(repos) + stderr(f" fetching READMEs ({total} repos)...") + for start in range(0, total, _BATCH_SIZE): + batch = repos[start : start + _BATCH_SIZE] + stderr(f" batch {start + 1}-{start + len(batch)}/{total}...") + repo_parts = [] + for i, c in enumerate(batch): + # Try both README.md and README.rst. + repo_parts.append( + f" repo{i}: repository(" + f'owner: "{c["owner"]}", ' + f'name: "{c["repo"]}") {{\n' + f" readmeMd: object(" + f'expression: "HEAD:README.md") {{\n' + f" ... on Blob {{ text }}\n" + f" }}\n" + f" readmeRst: object(" + f'expression: "HEAD:README.rst") {{\n' + f" ... on Blob {{ text }}\n" + f" }}\n" + f" }}" + ) + query = "query {\n" + "\n".join(repo_parts) + "\n}" + data = graphql(session, query) + if data is None: + for c in batch: + result[c["full_name"]] = "" + continue + for i, c in enumerate(batch): + repo_data = data.get(f"repo{i}") + text = "" + if repo_data: + for key in ("readmeMd", "readmeRst"): + obj = repo_data.get(key) + if obj and "text" in obj: + text = obj["text"] + break + result[c["full_name"]] = text + return result + + +# --- Fetch dep files --- + + +def _build_fixed_dep_fragment(): + """Build GraphQL fields for fetching fixed dep files + + requirements/ dir listing. + """ + fields = [] + for dep_file, alias in _FIXED_DEP_FILES.items(): + fields.append( + f" {alias}: object(" + f'expression: "HEAD:{dep_file}") {{\n' + f" ... on Blob {{ text }}\n" + f" }}" + ) + # Also fetch the requirements/ directory listing. + fields.append( + " requirementsDir: object(" + "expression: \"HEAD:requirements\") {\n" # noqa: Q003 + " ... on Tree { entries { name } }\n" + " }" + ) + return "\n".join(fields) + + +def _make_req_alias(filename): + """Turn a requirements/*.txt filename into a GraphQL alias.""" + base = filename.replace(".txt", "") + base = re.sub(r"[^a-zA-Z0-9]", "_", base) + return f"req_{base}" + + +def fetch_dep_files(session, repos): + """Batch-fetch dependency files for a list of repos. + + Phase 1: fetch fixed dep files + requirements/ dir listing. + Phase 2: for repos with a requirements/ dir, fetch all *.txt + files found there. + + Returns a dict mapping full_name to a dict of + {dep_file: content}. + """ + fixed_fragment = _build_fixed_dep_fragment() + result = {} + # Track which repos have requirements/ entries. + req_dir_entries = {} # full_name -> [filename, ...] + + # --- Phase 1: fixed files + dir listing --- + total = len(repos) + stderr( + " phase 1: fixed files + requirements/ dir listing " + f"({total} repos)..." + ) + for start in range(0, total, _BATCH_SIZE): + batch = repos[start : start + _BATCH_SIZE] + stderr(f" batch {start + 1}-{start + len(batch)}/{total}...") + repo_parts = [] + for i, c in enumerate(batch): + repo_parts.append( + f" repo{i}: repository(" + f'owner: "{c["owner"]}", ' + f'name: "{c["repo"]}") {{\n' + f"{fixed_fragment}\n" + f" }}" + ) + query = "query {\n" + "\n".join(repo_parts) + "\n}" + data = graphql(session, query) + if data is None: + for c in batch: + result[c["full_name"]] = {} + continue + for i, c in enumerate(batch): + repo_data = data.get(f"repo{i}") + files = {} + if repo_data: + for dep_file, alias in _FIXED_DEP_FILES.items(): + obj = repo_data.get(alias) + if obj and "text" in obj: + files[dep_file] = obj["text"] + # Check for requirements/ directory. + req_obj = repo_data.get("requirementsDir") + if req_obj and "entries" in req_obj: + txt_files = [ + e["name"] + for e in req_obj["entries"] + if e["name"].endswith(".txt") + ] + if txt_files: + req_dir_entries[c["full_name"]] = txt_files + result[c["full_name"]] = files + + # --- Phase 2: fetch discovered requirements/*.txt files --- + if req_dir_entries: + # Collect repos that need follow-up. + need_fetch = [c for c in repos if c["full_name"] in req_dir_entries] + stderr( + " phase 2: fetching requirements/*.txt from " + f"{len(need_fetch)} repos..." + ) + for start in range(0, len(need_fetch), _BATCH_SIZE): + batch = need_fetch[start : start + _BATCH_SIZE] + repo_parts = [] + for i, c in enumerate(batch): + txt_files = req_dir_entries[c["full_name"]] + file_fields = [] + for fname in txt_files: + alias = _make_req_alias(fname) + path = f"requirements/{fname}" + file_fields.append( + f" {alias}: object(" + f'expression: "HEAD:{path}") {{\n' + f" ... on Blob {{ text }}\n" + f" }}" + ) + repo_parts.append( + f" repo{i}: repository(" + f'owner: "{c["owner"]}", ' + f'name: "{c["repo"]}") {{\n' + + "\n".join(file_fields) + + "\n }" + ) + query = "query {\n" + "\n".join(repo_parts) + "\n}" + data = graphql(session, query) + if data is None: + continue + for i, c in enumerate(batch): + repo_data = data.get(f"repo{i}") + if not repo_data: + continue + txt_files = req_dir_entries[c["full_name"]] + for fname in txt_files: + alias = _make_req_alias(fname) + obj = repo_data.get(alias) + if obj and "text" in obj: + path = f"requirements/{fname}" + result[c["full_name"]][path] = obj["text"] + + return result + + +# --- Classify --- + + +def classify_dependency(file_contents): + """Classify the dependency type from fetched file contents. + + Returns a tuple (status, detail) where status is one of: + - "direct" : PROJECT in install/runtime dependencies + - "build" : PROJECT in build/setup dependencies only + - "test" : PROJECT in test/dev dependencies only + - "optional" : PROJECT in optional/extras dependencies + - "no" : not found in any dependency file + """ + pat = re.escape(PROJECT) + found_in = [] + for dep_file, content in file_contents.items(): + if content is None: + continue + if not re.search(r"\b" + pat + r"\b", content): + continue + found_in.append(dep_file) + + if not found_in: + return "no", "" + + # Classify the dependency type based on which files it was + # found in. + for f in found_in: + content = file_contents[f] + if f == "pyproject.toml": + if re.search( + r"\[project\].*?dependencies\s*=\s*\[.*?" + pat, + content, + re.DOTALL, + ): + return "direct", f + if re.search( + r"\[tool\.poetry\.dependencies\].*?" + pat, + content, + re.DOTALL, + ): + return "direct", f + if re.search( + r"\[build-system\].*?requires\s*=\s*\[.*?" + pat, + content, + re.DOTALL, + ): + return "build", f + if r"optional-dependencies" in content: + return "optional", f + if re.search(r"test|dev", content): + return "test", f + return "direct", f + elif f == "setup.py": + if re.search(r"install_requires.*?" + pat, content, re.DOTALL): + return "direct", f + if re.search(r"setup_requires.*?" + pat, content, re.DOTALL): + return "build", f + if re.search(r"tests_require.*?" + pat, content, re.DOTALL): + return "test", f + if re.search(r"extras_require.*?" + pat, content, re.DOTALL): + return "optional", f + return "direct", f + elif f == "setup.cfg": + if re.search(r"install_requires.*?" + pat, content, re.DOTALL): + return "direct", f + if re.search(r"extras_require.*?" + pat, content, re.DOTALL): + return "optional", f + return "direct", f + elif "requirements" in f: + return "direct", f + + return "direct", ", ".join(found_in) + + +# --- Misc --- + + +def make_subst_name(full_name): + """Turn 'owner/repo' into a substitution-safe base name.""" + name = full_name.split("/")[1] + # Replace underscores and dots with hyphens. + name = re.sub(r"[_.]", "-", name) + return name.lower() + + +def tier_label(stars): + if stars >= 40000: + return 1 + elif stars >= 10000: + return 2 + else: + return 3 + + +def generate_rst(projects): + """Generate RST output for adoption.rst.""" + tiers = {1: [], 2: [], 3: []} + for p in projects: + t = tier_label(p["stars"]) + tiers[t].append(p) + + lines = [] + star_badges = [] + logo_images = [] + + tier_headers = { + 1: "Tier 1 (>40k GitHub stars)", + 2: "Tier 2 (10k-40k GitHub stars)", + 3: "Tier 3 (1k-10k GitHub stars)", + } + + for tier_num in (1, 2, 3): + tier_projects = sorted(tiers[tier_num], key=lambda x: -x["stars"]) + if not tier_projects: + continue + + header = tier_headers[tier_num] + lines.extend([ + header, + "-" * len(header), + "", + ".. list-table::", + " :header-rows: 1", + " :widths: 18 42 12 28", + "", + " * - Project", + " - Description", + " - Stars", + " - Usage", + ]) + + for p in tier_projects: + name = make_subst_name(p["full_name"]) + owner = p["owner"] + repo = p["repo"] + full = p["full_name"] + desc = p["description"] + # Truncate description to fit RST table. + if len(desc) > 60: + desc = desc[:57] + "..." + dep_type = p.get("dep_type", "") + usage = "" + if dep_type == "build": + usage = "build-time dependency" + elif dep_type == "test": + usage = "test dependency" + elif dep_type == "optional": + usage = "optional dependency" + + proj_link = f"|{name}-logo| `{repo} `__" + lines.extend([ + f" * - {proj_link}", + f" - {desc}", + f" - |{name}-stars|", + f" - {usage}", + ]) + + star_badges.append( + f".. |{name}-stars| image:: " + "https://img.shields.io/github/stars/" + f"{full}.svg?style=plastic" + ) + + logo_images.append( + f".. |{name}-logo| image:: " + f"https://github.com/{owner}.png?s=28 :height: 28" + ) + + lines.append("") + + # Combine everything. + output = [] + output.extend(lines) + output.extend([ + "", + ".. Star badges", + "", + ]) + output.extend(star_badges) + output.extend([ + "", + ".. Logo images", + "", + ]) + output.extend(logo_images) + return "\n".join(output) + + +# --- Cache --- + + +def load_cache(): + """Load cached data from disk. + + Returns a dict with keys: min_stars, repos, readmes, dep_files. + Returns None if cache is missing, stale, or --no-cache is set. + The cache is invalidated if the current MIN_STARS is lower + than the min_stars used to build the cache (we'd be missing + repos). + """ + if NO_CACHE: + return None + if not os.path.exists(_CACHE_FILE): + return None + try: + with open(_CACHE_FILE, "rb") as f: + data = pickle.load(f) + except (OSError, pickle.UnpicklingError, EOFError) as err: + stderr(f" cache load error: {err}") + return None + cached_min_stars = data.get("min_stars", 0) + if cached_min_stars > MIN_STARS: + stderr( + f" cache built with min_stars={cached_min_stars}, " + f"but current min_stars={MIN_STARS}; ignoring cache" + ) + return None + stderr( + f" loaded cache ({len(data.get('repos', []))} repos, " + f"min_stars={cached_min_stars})" + ) + return data + + +def save_cache(repos, readmes, dep_files): + """Save fetched data to disk.""" + data = { + "min_stars": MIN_STARS, + "repos": repos, + "readmes": readmes, + "dep_files": dep_files, + } + with open(_CACHE_FILE, "wb") as f: + pickle.dump(data, f) + stderr(f" saved cache to {_CACHE_FILE}") + + +# --- CLI --- + + +def parse_cli(): + """Parse CLI arguments and set global constants.""" + global PROJECT, MIN_STARS, MAX_STARS, TOKEN, SKIP, INREADME, INDEPS, NO_CACHE # noqa: E501 + + parser = argparse.ArgumentParser( + description=( + "Find notable GitHub projects using a given " + "project as a dependency." + ) + ) + parser.add_argument( + "--project", + required=True, + help="Project name to search for (e.g. 'psutil').", + ) + parser.add_argument( + "--min-stars", + type=int, + default=300, + help="Minimum GitHub stars to consider (default: 300).", + ) + parser.add_argument( + "--max-stars", + type=int, + default=0, + help="Maximum GitHub stars (default: no limit).", + ) + parser.add_argument( + "--token", + required=True, + help="Path to a file containing the GitHub token.", + ) + parser.add_argument( + "--skip", + nargs="*", + default=[], + help="Repos URLs to skip.", + ) + parser.add_argument( + "--skip-file-urls", + default=None, + help="Path to file with GitHub repo URLs to skip (found via regex).", + ) + parser.add_argument( + "--inreadme", + action="store_true", + default=False, + help="Filter: PROJECT must be mentioned in README.", + ) + parser.add_argument( + "--indeps", + action="store_true", + default=False, + help=( + "Filter: PROJECT must be mentioned in dep files " + "(pyproject.toml, setup.py, setup.cfg, " + "requirements*.txt)." + ), + ) + parser.add_argument( + "--no-cache", + action="store_true", + default=False, + help="Force fresh fetch, ignoring cached data.", + ) + args = parser.parse_args() + + if not args.inreadme and not args.indeps: + parser.error("at least one of --inreadme, --indeps is required") + + PROJECT = args.project + MIN_STARS = args.min_stars + MAX_STARS = args.max_stars + with open(os.path.expanduser(args.token)) as f: + TOKEN = f.read().strip() + INREADME = args.inreadme + INDEPS = args.indeps + + SKIP = set(args.skip) + SKIP.add("https://github.com/vinta/awesome-python") + NO_CACHE = args.no_cache + if args.skip_file_urls: + path = args.skip_file_urls + with open(path) as f: + text = f.read() + urls = [ + m.group(1) + for m in re.finditer( + r"(https://github\.com/[\w.-]+/[\w.-]+)", text + ) + ] + SKIP.update(urls) + + +# --- Main --- + + +def main(): + parse_cli() + session = get_session(TOKEN) + + cached = load_cache() + if cached is not None: + repos = cached["repos"] + readmes = cached.get("readmes", {}) + dep_files = cached.get("dep_files", {}) + need_save = False + else: + repos = None + readmes = {} + dep_files = {} + need_save = True + + # Step 1: enumerate all popular Python repos. + if repos is None: + stderr(f"Enumerating Python repos (>={MIN_STARS} stars)...") + repos = enumerate_repos(session) + stderr(f"Found {len(repos)} repos.") + + # Filter out skipped and archived repos. + filtered = [] + for repo in repos: + if repo["html_url"] in SKIP: + print(f"skipping {yellow(repo['html_url'])}") + continue + if repo["archived"]: + print(f"skipping {yellow(repo['html_url'])}") + continue + filtered.append(repo) + active_repos = filtered + stderr(f"After skip/archive filtering: {len(active_repos)} repos.") + + # Step 2: fetch data needed for the requested filters. + # Only fetch what's missing from cache. + if INREADME and not readmes: + stderr("Fetching READMEs...") + readmes = fetch_readmes(session, active_repos) + need_save = True + if INDEPS and not dep_files: + stderr("Fetching dependency files...") + dep_files = fetch_dep_files(session, active_repos) + need_save = True + + if need_save: + save_cache(repos, readmes, dep_files) + + # Step 3: apply filters. + confirmed = [] + for repo in active_repos: + name = repo["full_name"] + pat = re.escape(PROJECT) + + if INREADME: + readme = readmes.get(name, "") + if not re.search(r"\b" + pat + r"\b", readme): + continue + + if INDEPS: + files = dep_files.get(name, {}) + status, detail = classify_dependency(files) + if status == "no": + continue + repo["dep_type"] = status + repo["dep_detail"] = detail + + confirmed.append(repo) + + stderr() + stderr(f"Confirmed {len(confirmed)} projects:") + for c in confirmed: + stderr(f" {c['stars']:,} stars: {green(c['html_url'])}") + + # Generate RST. + if confirmed: + ans = input("\nGenerate RsT content? [y/N] ").strip().lower() + if ans in {"y", "yes"}: + rst = generate_rst(confirmed) + print(rst) + + +if __name__ == "__main__": + main() diff --git a/scripts/internal/docs/new_blog_post.py b/scripts/internal/docs/new_blog_post.py new file mode 100755 index 0000000000..b887bf9ad5 --- /dev/null +++ b/scripts/internal/docs/new_blog_post.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Create a skeleton RST file for a new blog post at +docs/blog//.rst. Invoke via `make blog-post` from docs/. +""" + +import argparse +import datetime +import pathlib +import sys + +HERE = pathlib.Path(__file__).resolve().parent +REPO_ROOT = HERE.parent.parent.parent +AUTHOR = "Giampaolo Rodola" + +SLUG = "" +TITLE = "" +TAGS = "" + + +def parse_cli(): + global SLUG, TITLE, TAGS + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "slug", help="URL slug, e.g. 'event-driven-process-waiting'" + ) + parser.add_argument( + "--title", + default=None, + help=( + "Post title. If omitted, derived from the slug, e.g. " + "'my-new-post' -> 'My new post'." + ), + ) + parser.add_argument( + "--tags", + default="", + help="Comma-separated tags, e.g. 'performance, linux'", + ) + args = parser.parse_args() + + SLUG = args.slug + TITLE = args.title or args.slug.replace("-", " ").capitalize() + TAGS = args.tags + + +SKELETON = """\ +.. post:: {date} + :tags: {tags} + :author: {author} + :exclude: + + One-line summary for listing cards and Atom feeds. + +{title} +{underline} + +Opening paragraph. + +Section +------- + +Body. +""" + + +def main(): + parse_cli() + today = datetime.date.today() + year_dir = REPO_ROOT / "docs" / "blog" / str(today.year) + year_dir.mkdir(parents=True, exist_ok=True) + path = year_dir / f"{SLUG}.rst" + if path.exists(): + print(f"error: {path} already exists", file=sys.stderr) + return 1 + content = SKELETON.format( + date=today.isoformat(), + tags=TAGS, + author=AUTHOR, + title=TITLE, + underline="=" * len(TITLE), + ) + path.write_text(content) + print(f"created: {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/internal/docs/refresh_adoption_stats.py b/scripts/internal/docs/refresh_adoption_stats.py new file mode 100755 index 0000000000..f0c6e3e5f4 --- /dev/null +++ b/scripts/internal/docs/refresh_adoption_stats.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola. All rights reserved. +# Use of this source code is governed by a BSD-style license that can +# be found in the LICENSE file. + +"""Refresh the dynamic numbers in docs/adoption.rst and README.rst +(monthly downloads and GitHub repository dependents). +""" + +import argparse +import json +import pathlib +import re +import sys +import time +import urllib.error +import urllib.request + +ROOT = pathlib.Path(__file__).resolve().parents[3] +TIMEOUT = 5 +RETRIES = 3 +BACKOFF = 5 +TARGETS = [ + ROOT / "docs/adoption.rst", + ROOT / "docs/index.rst", + ROOT / "README.rst", +] +DEPENDENTS_URL = "https://github.com/giampaolo/psutil/network/dependents" +DOWNLOADS_URL = "https://pypistats.org/api/packages/psutil/recent" + + +def fetch(url, accept="text/html"): + req = urllib.request.Request( + url, + headers={ + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0 Safari/537.36" + ), + "Accept": accept, + }, + ) + for attempt in range(1, RETRIES + 1): + try: + with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: + return resp.read().decode("utf-8", errors="replace") + except urllib.error.HTTPError as err: + if err.code != 429 or attempt == RETRIES: + sys.exit( + f"error: {url} returned HTTP {err.code} ({err.reason})" + ) + wait = int(err.headers.get("Retry-After") or BACKOFF * attempt) + print(f" rate limited, retrying in {wait}s ...", file=sys.stderr) + time.sleep(wait) + + +def round_millions(n): + """338018876 -> '330+ million'. Floor to 10M.""" + millions = (n // 10_000_000) * 10 + return f"{millions}+ million" + + +def floor_to(n, bucket): + """Floor n to a multiple of `bucket`. 769412, 10_000 -> 760000.""" + return (n // bucket) * bucket + + +def fetch_monthly_downloads(): + data = json.loads(fetch(DOWNLOADS_URL, accept="application/json")) + return data["data"]["last_month"] + + +def fetch_github_dependents(): + """Scrape the 'Used by' repository count from GitHub's dependents + graph. + """ + html = fetch(DEPENDENTS_URL) + repos_re = re.search(r"([\d,]+)\s+Repositories", html) + if not repos_re: + sys.exit("could not parse GitHub dependents page") + return int(repos_re.group(1).replace(",", "")) + + +def parse_cli(): + parser = argparse.ArgumentParser( + description="Refresh adoptions dynamic numbers" + ) + parser.parse_args() + + +def main(): + parse_cli() + print(f"fetching {DOWNLOADS_URL} ...") + monthly = fetch_monthly_downloads() + print(f" monthly downloads: {monthly:,}") + + print(f"fetching {DEPENDENTS_URL} ...") + repos = fetch_github_dependents() + print(f" repos: {repos:,}") + + new_downloads = round_millions(monthly) # "330+ million" + new_repos = f"{floor_to(repos, 10_000):,}+" # "760,000+" + + subs = [ + # adoption.rst / README.rst: **330+ million** downloads ... + ( + re.compile(r"\*\*\d+\+\s+million\*\*"), + f"**{new_downloads}**", + ), + ( + re.compile(r"\*\*[\d,]+\+\*\*(?=\s+`?GitHub repositories)"), + f"**{new_repos}**", + ), + # index.rst home-stats banner. + ( + re.compile( + r'()\d+\+\s+million()' + ), + rf"\g<1>{new_downloads}\g<2>", + ), + ( + re.compile( + r'()[\d,]+\+(\s*\n\s*' + r'GitHub)' + ), + rf"\g<1>{new_repos}\g<2>", + ), + ] + + totals = [0] * len(subs) + for path in TARGETS: + text = path.read_text() + new_text = text + for i, (pat, repl) in enumerate(subs): + new_text, n = pat.subn(repl, new_text) + totals[i] += n + if new_text == text: + print(f" {path.relative_to(ROOT)}: already current") + else: + path.write_text(new_text) + print(f" {path.relative_to(ROOT)}: updated") + + for i, (pat, _) in enumerate(subs): + if totals[i] == 0: + sys.exit( + f"no file matched {pat.pattern!r} " + "(expected at least 1 match across all targets)" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/internal/docs/rst_unused_targets.py b/scripts/internal/docs/rst_unused_targets.py new file mode 100755 index 0000000000..b9bac47f9e --- /dev/null +++ b/scripts/internal/docs/rst_unused_targets.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola. All rights reserved. +# Use of this source code is governed by a BSD-style license that can +# be found in the LICENSE file. + +"""Check .rst files for URL hyperlink targets that are defined but +never referenced. + +Undefined references (backtick refs pointing at a missing target) are +already caught by Sphinx during the docs build, so this script only +covers the unused-target case. +""" + +import argparse +import re +import sys + +# .. _`Foo`: https://... or .. _foo: https://... +RE_URL_TARGET = re.compile(r"^\.\. _`?([^`\n:]+)`?:\s*https?://", re.MULTILINE) + +# `Foo Bar`_ but NOT `text `_ and NOT `text`__ +RE_BACKTICK_REF = re.compile(r"`([^`<\n]+)`_(?!_)") + +# bare ref: BPO-12442_ +RE_BARE_REF = re.compile(r"(? due to inefficient + regex. +* Remove duplicates (because regex is not 100% efficient as of now). +* Check validity of URL, using HEAD request. (HEAD to save bandwidth) + Uses requests module for others are painful to use. REFERENCES[9] + Handles redirects, http, https, ftp as well. + +REFERENCES: +Using [1] with some modifications for including ftp +[1] http://stackoverflow.com/a/6883094/5163807 +[2] http://stackoverflow.com/a/31952097/5163807 +[3] http://daringfireball.net/2010/07/improved_regex_for_matching_urls +[4] https://mathiasbynens.be/demo/url-regex +[5] https://github.com/django/django/blob/master/django/core/validators.py +[6] https://data.iana.org/TLD/tlds-alpha-by-domain.txt +[7] https://codereview.stackexchange.com/questions/19663/http-url-validating +[8] https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD +[9] http://docs.python-requests.org/ + +Author: Himanshu Shekhar (2017) +""" + +import argparse +import concurrent.futures +import functools +import os +import re +import sys +import traceback + +import requests + +REGEX = re.compile( + r'(?:http|ftp|https)?://' + r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' +) +REQUEST_TIMEOUT = 15 +# There are some status codes sent by websites on HEAD request. +# Like 503 by Microsoft, and 401 by Apple +# They need to be sent GET request +RETRY_STATUSES = [503, 401, 403] + + +def sanitize_url(url): + url = url.rstrip(',') + url = url.rstrip('.') + url = url.lstrip('(') + url = url.rstrip(')') + url = url.lstrip('[') + url = url.rstrip(']') + url = url.lstrip('<') + url = url.rstrip('>') + return url + + +def find_urls(s): + matches = REGEX.findall(s) or [] + return list({sanitize_url(x) for x in matches}) + + +def parse_rst(fname): + """Look for links in a .rst file.""" + with open(fname) as f: + text = f.read() + return find_urls(text) + + +def parse_py(fname): + """Look for links in a .py file.""" + with open(fname) as f: + lines = f.readlines() + urls = set() + for i, line in enumerate(lines): + for url in find_urls(line): + # comment block + if line.lstrip().startswith('# '): + subidx = i + 1 + while True: + nextline = lines[subidx].strip() + if re.match(r"^# .+", nextline): + url += nextline[1:].strip() + else: + break + subidx += 1 + urls.add(url) + return list(urls) + + +def parse_c(fname): + """Look for links in a .py file.""" + with open(fname) as f: + lines = f.readlines() + urls = set() + for i, line in enumerate(lines): + for url in find_urls(line): + # comment block // + if line.lstrip().startswith('// '): + subidx = i + 1 + while True: + nextline = lines[subidx].strip() + if re.match(r"^// .+", nextline): + url += nextline[2:].strip() + else: + break + subidx += 1 + # comment block /* + elif line.lstrip().startswith('* '): + subidx = i + 1 + while True: + nextline = lines[subidx].strip() + if re.match(r'^\* .+', nextline): + url += nextline[1:].strip() + else: + break + subidx += 1 + urls.add(url) + return list(urls) + + +def parse_generic(fname): + with open(fname, errors='ignore') as f: + text = f.read() + return find_urls(text) + + +def get_urls(fname): + """Extracts all URLs in fname and return them as a list.""" + if fname.endswith('.rst'): + return parse_rst(fname) + elif fname.endswith('.py'): + return parse_py(fname) + elif fname.endswith(('.c', '.h')): + return parse_c(fname) + else: + with open(fname, errors='ignore') as f: + if f.readline().strip().startswith('#!/usr/bin/env python3'): + return parse_py(fname) + return parse_generic(fname) + + +@functools.lru_cache +def validate_url(url): + """Validate the URL by attempting an HTTP connection. + Makes an HTTP-HEAD request for each URL. + """ + try: + res = requests.head(url, timeout=REQUEST_TIMEOUT) + # some websites deny 503, like Microsoft + # and some send 401, like Apple, observations + if (not res.ok) and (res.status_code in RETRY_STATUSES): + res = requests.get(url, timeout=REQUEST_TIMEOUT) + return res.ok + except requests.exceptions.RequestException: + return False + + +def parallel_validator(urls): + """Validates all urls in parallel + urls: tuple(filename, url). + """ + fails = [] # list of tuples (filename, url) + current = 0 + total = len(urls) + with concurrent.futures.ThreadPoolExecutor() as executor: + fut_to_url = { + executor.submit(validate_url, url[1]): url for url in urls + } + for fut in concurrent.futures.as_completed(fut_to_url): + current += 1 + sys.stdout.write(f"\r{current} / {total}") + sys.stdout.flush() + fname, url = fut_to_url[fut] + try: + ok = fut.result() + except Exception: # noqa: BLE001 + fails.append((fname, url)) + print() + print(f"warn: error while validating {url}", file=sys.stderr) + traceback.print_exc() + else: + if not ok: + fails.append((fname, url)) + + print() + return fails + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter + ) + parser.add_argument('files', nargs="+") + parser.parse_args() + args = parser.parse_args() + + all_urls = [] + for fname in args.files: + urls = get_urls(fname) + if urls: + print(f"{len(urls):4} {fname}") + all_urls.extend((fname, url) for url in urls) + + fails = parallel_validator(all_urls) + if not fails: + print("all links are valid; cheers!") + else: + for fail in fails: + fname, url = fail + print(f"{fname:<30}: {url} ") + print('-' * 20) + print(f"total: {len(fails)} fails!") + sys.exit(1) + + +if __name__ == '__main__': + try: + main() + except (KeyboardInterrupt, SystemExit): + os._exit(0) diff --git a/scripts/internal/generate_manifest.py b/scripts/internal/generate_manifest.py new file mode 100755 index 0000000000..f6549ec58c --- /dev/null +++ b/scripts/internal/generate_manifest.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Generate MANIFEST.in file.""" + +import os +import shlex +import subprocess + +SKIP_EXTS = ('.png', '.jpg', '.jpeg') +SKIP_FILES = () +SKIP_PREFIXES = ('.github/', 'docs/') + + +def sh(cmd): + return subprocess.check_output( + shlex.split(cmd), universal_newlines=True + ).strip() + + +def main(): + files = set() + for file in sh("git ls-files").split('\n'): + if ( + file.startswith(SKIP_PREFIXES) + or os.path.splitext(file)[1].lower() in SKIP_EXTS + or file in SKIP_FILES + ): + continue + files.add(file) + + for file in sorted(files): + print("include " + file) + + +if __name__ == '__main__': + main() diff --git a/scripts/internal/git_pre_commit.py b/scripts/internal/git_pre_commit.py new file mode 100755 index 0000000000..b98eab768d --- /dev/null +++ b/scripts/internal/git_pre_commit.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""This gets executed on 'git commit' and rejects the commit in case +the submitted code does not pass validation. Validation is run only +against the files which were modified in the commit. +""" + +import os +import pathlib +import shlex +import subprocess +import sys + +ROOT_DIR = pathlib.Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(ROOT_DIR)) +from _bootstrap import load_module # noqa: E402 + +_common = load_module(ROOT_DIR / "psutil" / "_common.py") +hilite = _common.hilite + +PYTHON = sys.executable + + +def log(msg="", color=None, bold=None): + msg = "Git pre-commit > " + msg + if msg: + msg = hilite(msg, color=color, bold=bold, force_color=True) + print(msg, flush=True) + + +def exit_with(msg): + log(msg + " Commit aborted.", color="red") + sys.exit(1) + + +def sh(cmd): + if isinstance(cmd, str): + cmd = shlex.split(cmd) + p = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = p.communicate() + if p.returncode != 0: + raise RuntimeError(stderr) + if stderr: + log(stderr) + return stdout.rstrip() + + +def git_commit_files(): + out = [ + f + for f in sh(["git", "diff", "--cached", "--name-only"]).splitlines() + if os.path.exists(f) + ] + + py = [f for f in out if f.endswith(".py")] + c = [f for f in out if f.endswith((".c", ".h"))] + rst = [f for f in out if f.endswith(".rst")] + toml = [f for f in out if f.endswith(".toml")] + new_rm_mv = sh( + ["git", "diff", "--name-only", "--diff-filter=ADR", "--cached"] + ).split() + return py, c, rst, toml, new_rm_mv + + +def lint_manifest(): + out = sh([PYTHON, "scripts/internal/generate_manifest.py"]) + with open("MANIFEST.in", encoding="utf8") as f: + if out.strip() != f.read().strip(): + exit_with( + "Some files were added, deleted or renamed. " + "Run 'make generate-manifest' and commit again." + ) + + +def run_make(target, files): + ls = ", ".join([os.path.basename(x) for x in files]) + plural = "s" if len(files) > 1 else "" + msg = f"Running 'make {target}' against {len(files)} file{plural}: {ls}" + log(msg, color="lightblue") + files = "FILES=" + " ".join(shlex.quote(f) for f in files) + if subprocess.call(["make", target, files]) != 0: + exit_with(f"'make {target}' failed.") + + +def main(): + py, c, rst, toml, new_rm_mv = git_commit_files() + if py: + run_make("black", py) + run_make("ruff", py) + if c: + run_make("lint-c", c) + if rst: + run_make("lint-rst", rst) + if toml: + run_make("lint-toml", toml) + if new_rm_mv: + lint_manifest() + + +if __name__ == "__main__": + main() diff --git a/scripts/internal/install-pydeps.sh b/scripts/internal/install-pydeps.sh new file mode 100755 index 0000000000..be1125eecd --- /dev/null +++ b/scripts/internal/install-pydeps.sh @@ -0,0 +1,90 @@ +#!/bin/sh + +# Install Python deps with uv, installing uv itself if missing. Falls back +# on pip where uv ships no binary for the platform (BSD, AIX, SunOS). +# NOTE: this script MUST be kept compatible with the `sh` shell. + +set -e + +if [ -z "$PYTHON" ]; then + PYTHON=python3 +fi + +UV= + +# --- pip + +install_pip() { + echo "installing pip" + "$PYTHON" "$(dirname "$0")/install_pip.py" +} + +pip_install() { + install_pip + echo "installing $* via pip" + PIP_BREAK_SYSTEM_PACKAGES=1 "$PYTHON" -m pip install \ + --upgrade \ + --upgrade-strategy eager \ + "$@" +} + +# --- uv + +find_uv() { + if command -v uv > /dev/null 2>&1; then + UV=$(command -v uv) + else + UV=$("$PYTHON" -c \ + 'from uv import find_uv_bin; print(find_uv_bin())' 2>/dev/null) || UV= + fi +} + +install_uv() { + install_pip || return + echo "installing uv" + # --only-binary: pick up the .whl, else fail + PIP_BREAK_SYSTEM_PACKAGES=1 "$PYTHON" -m pip install \ + --only-binary=:all: \ + --upgrade \ + 'uv>=0.8.8' +} + +uv_install() { + echo "installing $* via uv" + # When the interpreter is not writable, pip falls back to the user + # base; uv does not, so check ourselves. --prefix is not exactly + # pip --user, but avoids requiring a venv or root. + user_base=$("$PYTHON" -c \ + 'import os, site, sysconfig +writable = os.access(sysconfig.get_path("purelib"), os.W_OK) +print("" if writable else site.getuserbase())') + if [ -n "$user_base" ]; then + set -- --prefix "$user_base" "$@" + fi + "$UV" pip install \ + --python "$("$PYTHON" -c 'import sys; print(sys.executable)')" \ + --upgrade \ + "$@" +} + +main() { + if [ $# -eq 0 ]; then + echo "usage: $0 " >&2 + exit 1 + fi + + find_uv + + if [ -z "$UV" ]; then + install_uv || echo "$0: uv unavailable on this platform, using pip" >&2 + find_uv + fi + + if [ -n "$UV" ]; then + uv_install "$@" + else + pip_install "$@" + fi +} + +main "$@" diff --git a/scripts/internal/install-sysdeps.sh b/scripts/internal/install-sysdeps.sh new file mode 100755 index 0000000000..7cd768018a --- /dev/null +++ b/scripts/internal/install-sysdeps.sh @@ -0,0 +1,115 @@ +#!/bin/sh + +# Install the system dependencies needed to compile psutil. With --test-only +# install CLI tools needed by unit tests. +# NOTE: this script MUST be kept compatible with the `sh` shell. + +set -e + +if [ "$1" = "--test-only" ]; then + TEST_ONLY=true +fi + +UNAME_S=$(uname -s) + +case "$UNAME_S" in + Linux) + if command -v apt-get > /dev/null 2>&1; then + HAS_APT=true # debian / ubuntu + elif command -v dnf > /dev/null 2>&1; then + RPM_MGR=dnf # fedora, redhat 8+ + elif command -v yum > /dev/null 2>&1; then + RPM_MGR=yum # older redhat / centos + elif command -v pacman > /dev/null 2>&1; then + HAS_PACMAN=true # arch + elif command -v apk > /dev/null 2>&1; then + HAS_APK=true # musl + fi + ;; + FreeBSD) + FREEBSD=true + ;; + NetBSD) + NETBSD=true + ;; + OpenBSD) + OPENBSD=true + ;; + SunOS) + SUNOS=true + ;; +esac + +# Check if running as root +if [ "$(id -u)" -ne 0 ]; then + SUDO=sudo +fi + +# Deps needed to compile psutil. +install_build_deps() { + # Debian / Ubuntu + if [ $HAS_APT ]; then + $SUDO apt-get install -y python3-dev gcc + # Redhat / Fedora + elif [ $RPM_MGR ]; then + $SUDO $RPM_MGR install -y python3-devel gcc + # Arch + elif [ $HAS_PACMAN ]; then + $SUDO pacman -S --noconfirm python gcc + # Alpine + elif [ $HAS_APK ]; then + $SUDO apk add --no-interactive python3-dev gcc musl-dev linux-headers + # FreeBSD + elif [ $FREEBSD ]; then + $SUDO pkg install -y python3 # no gcc: base cc is clang, and that's what python uses + # NetBSD + elif [ $NETBSD ]; then + PKGIN=/usr/pkg/bin/pkgin + if [ ! -x "$PKGIN" ]; then + : "${PKG_PATH:=https://cdn.netbsd.org/pub/pkgsrc/packages/NetBSD/$(uname -m)/$(uname -r)/All}" + $SUDO env PKG_PATH="$PKG_PATH" /usr/sbin/pkg_add -v pkgin + fi + $SUDO "$PKGIN" update + $SUDO "$PKGIN" -y install 'python314-*' # no gcc12: base gcc compiles psutil just fine + if [ ! -e /usr/pkg/bin/python3 ]; then + $SUDO ln -s /usr/pkg/bin/python3.14 /usr/pkg/bin/python3 + fi + # OpenBSD + elif [ $OPENBSD ]; then + $SUDO pkg_add python%3 # there's no "python3" package, and no gcc: base cc is clang + # SunOS + elif [ $SUNOS ]; then + $SUDO pkg install developer/gcc + else + echo "Unsupported platform '$UNAME_S'. Ignoring." + fi +} + +# CLI tools needed by unit tests. +install_test_deps() { + # Debian / Ubuntu + if [ $HAS_APT ]; then + $SUDO apt-get install -y net-tools coreutils util-linux sudo procps + # Redhat / Fedora + elif [ $RPM_MGR ]; then + $SUDO $RPM_MGR install -y net-tools util-linux sudo procps-ng + # Arch + elif [ $HAS_PACMAN ]; then + $SUDO pacman -S --noconfirm net-tools coreutils util-linux sudo procps-ng + # Alpine + elif [ $HAS_APK ]; then + $SUDO apk add --no-interactive coreutils util-linux procps + else + echo "No supported package manager found on '$UNAME_S'. Ignoring." + fi +} + +main() { + if [ $TEST_ONLY ]; then + install_test_deps + else + install_build_deps + fi +} + +main diff --git a/scripts/internal/install_pip.py b/scripts/internal/install_pip.py new file mode 100755 index 0000000000..1ffdbd3d56 --- /dev/null +++ b/scripts/internal/install_pip.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Install pip, or upgrade it if it's too old. + +Note: we build wheels on Python 3.8 (the floor), but don't run tests +for it, nor installs deps, so this script is never called there. +""" + +import re +import ssl +import subprocess +import sys +import tempfile +from urllib.request import urlopen + +try: + import pip +except ImportError: + pip = None + + +if sys.version_info >= (3, 10): + URL = "https://bootstrap.pypa.io/get-pip.py" +else: + URL = "https://bootstrap.pypa.io/pip/{}.{}/get-pip.py".format( + *sys.version_info[:2] + ) + +# Needed by "pip install --group" (PEP 735), used by the +# install-pydeps-* makefile targets. +MIN_VERSION = (25, 1) + + +def get_pip_version(): + if pip is not None: + match = re.match(r"(\d+)\.(\d+)", pip.__version__) + return (int(match.group(1)), int(match.group(2))) if match else (0, 0) + + +def install_pip(): + ssl_context = ( + ssl._create_unverified_context() + if hasattr(ssl, "_create_unverified_context") + else None + ) + opts = ["--upgrade", "--break-system-packages"] + if not hasattr(sys, "real_prefix") and sys.base_prefix == sys.prefix: + opts.append("--user") # rejected when inside a virtualenv + + with tempfile.NamedTemporaryFile(suffix=".py") as f: + print(f"downloading {URL} into {f.name}") + kwargs = dict(context=ssl_context) if ssl_context else {} + req = urlopen(URL, **kwargs) + data = req.read() + req.close() + + f.write(data) + f.flush() + print("download finished, installing pip") + + code = subprocess.call([sys.executable, f.name, *opts]) + + sys.exit(code) + + +def main(): + version = get_pip_version() + if version is None: + print("pip is not installed") + elif version < MIN_VERSION: + print(f"pip {pip.__version__} is too old; upgrading") + else: + print(f"pip (version {pip.__version__}) already installed") + return + install_pip() + + +if __name__ == "__main__": + main() diff --git a/scripts/internal/print_access_denied.py b/scripts/internal/print_access_denied.py new file mode 100755 index 0000000000..1633d6198c --- /dev/null +++ b/scripts/internal/print_access_denied.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Helper script iterates over all processes and . +It prints how many AccessDenied exceptions are raised in total and +for what Process method. + +$ make print-access-denied +API AD Percent Outcome +memory_info 0 0.0% SUCCESS +uids 0 0.0% SUCCESS +cmdline 0 0.0% SUCCESS +create_time 0 0.0% SUCCESS +status 0 0.0% SUCCESS +num_ctx_switches 0 0.0% SUCCESS +username 0 0.0% SUCCESS +ionice 0 0.0% SUCCESS +memory_percent 0 0.0% SUCCESS +gids 0 0.0% SUCCESS +cpu_times 0 0.0% SUCCESS +nice 0 0.0% SUCCESS +pid 0 0.0% SUCCESS +cpu_percent 0 0.0% SUCCESS +num_threads 0 0.0% SUCCESS +cpu_num 0 0.0% SUCCESS +ppid 0 0.0% SUCCESS +terminal 0 0.0% SUCCESS +name 0 0.0% SUCCESS +threads 0 0.0% SUCCESS +cpu_affinity 0 0.0% SUCCESS +memory_maps 71 21.3% ACCESS DENIED +memory_footprint 71 21.3% ACCESS DENIED +exe 174 52.1% ACCESS DENIED +environ 238 71.3% ACCESS DENIED +num_fds 238 71.3% ACCESS DENIED +io_counters 238 71.3% ACCESS DENIED +cwd 238 71.3% ACCESS DENIED +connections 238 71.3% ACCESS DENIED +open_files 238 71.3% ACCESS DENIED +-------------------------------------------------- +Totals: access-denied=1744, calls=10020, processes=334 +""" + +import time +from collections import defaultdict + +import psutil +from psutil._common import print_color + + +def main(): + # collect + tot_procs = 0 + tot_ads = 0 + tot_calls = 0 + signaler = object() + d = defaultdict(int) + start = time.time() + for p in psutil.process_iter(attrs=[], ad_value=signaler): + tot_procs += 1 + for methname, value in p.info.items(): + tot_calls += 1 + if value is signaler: + tot_ads += 1 + d[methname] += 1 + else: + d[methname] += 0 + elapsed = time.time() - start + + # print + templ = "{:<20} {:<5} {:<9} {}" + s = templ.format("API", "AD", "Percent", "Outcome") + print_color(s, color=None, bold=True) + for methname, ads in sorted(d.items(), key=lambda x: (x[1], x[0])): + perc = (ads / tot_procs) * 100 + outcome = "SUCCESS" if not ads else "ACCESS DENIED" + s = templ.format(methname, ads, f"{perc:6.1f}%", outcome) + print_color(s, "red" if ads else None) + tot_perc = round((tot_ads / tot_calls) * 100, 1) + print("-" * 50) + print( + f"Totals: access-denied={tot_ads} ({tot_perc}%%), calls={tot_calls}," + f" processes={tot_procs}, elapsed={round(elapsed, 2)}s" + ) + + +if __name__ == '__main__': + main() diff --git a/scripts/internal/print_announce.py b/scripts/internal/print_announce.py new file mode 100755 index 0000000000..af3e7601b8 --- /dev/null +++ b/scripts/internal/print_announce.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Prints release announce based on docs/changelog.rst file content. +See: https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode. + +""" + +import pathlib +import re +import subprocess +import sys + +from psutil import __version__ + +ROOT_DIR = pathlib.Path(__file__).resolve().parent.parent.parent +CHANGELOG = ROOT_DIR / 'docs' / 'changelog.rst' +PRINT_HASHES_PY = ROOT_DIR / 'scripts' / 'internal' / 'print_hashes.py' + +PRJ_NAME = 'psutil' +PRJ_VERSION = __version__ +PRJ_URL_HOME = 'https://github.com/giampaolo/psutil' +PRJ_URL_DOC = 'https://psutil.io' +PRJ_URL_DOWNLOAD = 'https://pypi.org/project/psutil/#files' +PRJ_URL_WHATSNEW = 'https://psutil.io/changelog/' + +template = """\ +Hello all, +I'm glad to announce the release of {prj_name} {prj_version}: +{prj_urlhome} + +About +===== + +psutil (process and system utilities) is a cross-platform library for \ +retrieving information on running processes and system utilization (CPU, \ +memory, disks, network) in Python. It is useful mainly for system \ +monitoring, profiling and limiting process resources and management of \ +running processes. It implements many functionalities offered by command \ +line tools such as: ps, top, lsof, netstat, ifconfig, who, df, kill, free, \ +nice, ionice, iostat, iotop, uptime, pidof, tty, taskset, pmap. It \ +currently supports Linux, Windows, macOS, Sun Solaris, FreeBSD, OpenBSD, \ +NetBSD and AIX. Supported Python versions are cPython 3.7+ and PyPy. + +What's new +========== + +{changes} + +Links +===== + +- Home page: {prj_urlhome} +- Download: {prj_urldownload} +- Documentation: {prj_urldoc} +- What's new: {prj_urlwhatsnew} + +Hashes +====== + +{hashes} + +-- + +Giampaolo - https://gmpy.dev/about +""" + + +def rst_to_text(s): + """Strip RST/Sphinx markup, returning plain text.""" + # :gh:`123` -> #123 + s = re.sub(r':gh:`(\d+)`', r'#\1', s) + # :meth:, :func:, :class:, :attr:, :exc:, :mod:, :data:, etc. + # :role:`text` -> text (also handles :role:`~text` and :role:`mod.text`) + s = re.sub(r':[a-z]+:`~?([^`]+)`', r'\1', s) + # ``code`` -> `code` + s = re.sub(r'``([^`]+)``', r'`\1`', s) + # **bold** -> bold + s = re.sub(r'\*\*([^*]+)\*\*', r'\1', s) + # *italic* -> italic + s = re.sub(r'\*([^*]+)\*', r'\1', s) + return s + + +def get_changes(): + """Get the most recent changes for this release by parsing + docs/changelog.rst file. + """ + with open(CHANGELOG) as f: + lines = f.readlines() + + block = [] + + # eliminate the part preceding the first block + while lines: + line = lines.pop(0) + if line.startswith('^^^^'): + break + else: + raise ValueError("something wrong") + + lines.pop(0) + while lines: + line = lines.pop(0) + line = line.rstrip() + if re.match(r"^- \d+_", line): + line = re.sub(r"^- (\d+)_", r"- #\1", line) + + if line.startswith('^^^^'): + break + block.append(line) + else: + raise ValueError("something wrong") + + # eliminate bottom empty lines + block.pop(-1) + while not block[-1]: + block.pop(-1) + + text = "\n".join(block) + text = rst_to_text(text) + return text + + +def main(): + changes = get_changes() + hashes = ( + subprocess.check_output([sys.executable, PRINT_HASHES_PY, 'dist/']) + .strip() + .decode() + ) + text = template.format( + prj_name=PRJ_NAME, + prj_version=PRJ_VERSION, + prj_urlhome=PRJ_URL_HOME, + prj_urldownload=PRJ_URL_DOWNLOAD, + prj_urldoc=PRJ_URL_DOC, + prj_urlwhatsnew=PRJ_URL_WHATSNEW, + changes=changes, + hashes=hashes, + ) + print(text) + + +if __name__ == '__main__': + main() diff --git a/scripts/internal/print_api_speed.py b/scripts/internal/print_api_speed.py new file mode 100755 index 0000000000..9cc7b7d56f --- /dev/null +++ b/scripts/internal/print_api_speed.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Benchmark all API calls and print them from fastest to slowest. + +$ make print_api_speed +SYSTEM APIS NUM CALLS SECONDS +------------------------------------------------- +getloadavg 300 0.00013 +heap_info 300 0.00028 +heap_trim 300 0.00039 +cpu_count 300 0.00061 +disk_usage 300 0.00066 +pid_exists 300 0.00235 +users 300 0.00455 +net_io_counters 300 0.00550 +cpu_times 300 0.00667 +boot_time 300 0.00700 +cpu_percent 300 0.00766 +net_if_stats 300 0.00783 +virtual_memory 300 0.00834 +cpu_times_percent 300 0.00885 +net_if_addrs 300 0.01157 +cpu_stats 300 0.01208 +swap_memory 300 0.01558 +disk_partitions 300 0.01664 +disk_io_counters 300 0.02204 +sensors_battery 300 0.02995 +pids 300 0.05295 +cpu_count (cores) 300 0.06943 +process_iter (all) 300 0.08486 +cpu_freq 300 0.18987 +sensors_fans 300 0.74027 +net_connections 161 2.00690 +sensors_temperatures 100 2.00742 + +PROCESS APIS NUM CALLS SECONDS +------------------------------------------------- +exe 300 0.00017 +create_time 300 0.00020 +nice 300 0.00025 +ionice 300 0.00041 +cwd 300 0.00052 +cpu_affinity 300 0.00059 +num_fds 300 0.00097 +memory_info 300 0.00201 +cmdline 300 0.00222 +io_counters 300 0.00226 +cpu_num 300 0.00242 +status 300 0.00242 +terminal 300 0.00243 +name 300 0.00249 +page_faults 300 0.00258 +memory_percent 300 0.00259 +cpu_times 300 0.00272 +threads 300 0.00278 +num_threads 300 0.00278 +gids 300 0.00296 +num_ctx_switches 300 0.00299 +uids 300 0.00311 +cpu_percent 300 0.00346 +net_connections 300 0.00373 +open_files 300 0.00378 +memory_extras 300 0.00398 +username 300 0.00500 +ppid 300 0.00556 +environ 300 0.01176 +memory_footprint 300 0.02218 +memory_maps 300 0.27158 +""" + +import argparse +import inspect +import os +import sys +from timeit import default_timer as timer + +import psutil +from psutil._common import print_color + +TIMES = 300 +PID = os.getpid() +timings = [] +templ = "{:<25} {:>10} {:>10}" + + +def print_header(what): + s = templ.format(what, "NUM CALLS", "SECONDS") + print_color(s, color=None, bold=True) + print("-" * len(s)) + + +def print_timings(): + timings.sort(key=lambda x: (x[1], -x[2]), reverse=True) + i = 0 + while timings[:]: + title, times, elapsed = timings.pop(0) + s = templ.format(title, str(times), f"{elapsed:.5f}") + if i > len(timings) - 5: + print_color(s, color="red") + else: + print(s) + + +def timecall(title, fun, *args, **kw): + print(f"{title:<50}", end="") + sys.stdout.flush() + t = timer() + for n in range(TIMES): + try: + fun(*args, **kw) + except psutil.AccessDenied: + return + else: + elapsed = timer() - t + if elapsed > 2: + break + print("\033[2K\r", end="") + sys.stdout.flush() + timings.append((title, n + 1, elapsed)) + + +def set_highest_priority(): + """Set highest CPU and I/O priority (requires root).""" + p = psutil.Process() + if psutil.WINDOWS: + p.nice(psutil.HIGH_PRIORITY_CLASS) + else: + p.nice(-20) + + if psutil.LINUX: + p.ionice(psutil.IOPRIO_CLASS_RT, value=7) + elif psutil.WINDOWS: + p.ionice(psutil.IOPRIO_HIGH) + + +def parse_cli(): + global TIMES, PID + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter + ) + parser.add_argument('-t', '--times', type=int, default=TIMES) + parser.add_argument('-p', '--pid', type=int, default=PID) + args = parser.parse_args() + TIMES = args.times + PID = args.pid + assert TIMES > 1, TIMES + + +def main(): + parse_cli() + + try: + set_highest_priority() + except psutil.AccessDenied: + prio_set = False + else: + prio_set = True + + # --- system + + public_apis = [] + ignore = [ + 'bytes2human', + 'wait_procs', + 'process_iter', + 'win_service_get', + 'win_service_iter', + ] + if psutil.MACOS: + ignore.append('net_connections') # raises AD + for name in psutil.__all__: + obj = getattr(psutil, name, None) + if inspect.isfunction(obj): + if name not in ignore: + public_apis.append(name) + + print_header("SYSTEM APIS") + for name in public_apis: + fun = getattr(psutil, name) + args = () + if name == 'pid_exists': + args = (PID,) + elif name == 'disk_usage': + args = (os.getcwd(),) + timecall(name, fun, *args) + timecall('cpu_count (cores)', psutil.cpu_count, logical=False) + timecall('process_iter (all)', lambda: list(psutil.process_iter())) + print_timings() + + # --- process + print() + print_header("PROCESS APIS") + p = psutil.Process(PID) + for name in sorted(p.attrs): + fun = getattr(p, name) + if callable(fun): + timecall(name, fun) + + print_timings() + + if not prio_set: + msg = "\nWARN: couldn't set highest process priority " + msg += "(requires root)" + print_color(msg, "red") + + +if __name__ == '__main__': + main() diff --git a/scripts/internal/print_dist.py b/scripts/internal/print_dist.py new file mode 100755 index 0000000000..2d4b82b5de --- /dev/null +++ b/scripts/internal/print_dist.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""List and pretty print tarball & wheel files in the dist/ directory.""" + +import argparse +import collections +import fnmatch +import os +import pathlib +import sys + +ROOT_DIR = pathlib.Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(ROOT_DIR)) +from _bootstrap import load_module # noqa: E402 + +_common = load_module(ROOT_DIR / "psutil" / "_common.py") +bytes2human = _common.bytes2human +print_color = _common.print_color + +# Tags are spelled out because they change silently when a different +# Python builds the wheel. Trailing manylinux ones follow the image. +EXPECTED_WHEELS = [ + # Linux + "*-cp38-abi3-manylinux2010_x86_64.*.whl", + "*-cp38-abi3-manylinux2014_aarch64.*.whl", + "*-cp38-abi3-manylinux2014_ppc64le.*.whl", + "*-cp38-abi3-manylinux2014_s390x.*.whl", + # Linux musl + "*-cp38-abi3-musllinux_1_2_x86_64.whl", + "*-cp38-abi3-musllinux_1_2_aarch64.whl", + # macOS + "*-cp38-abi3-macosx_10_15_x86_64.whl", + "*-cp38-abi3-macosx_11_0_arm64.whl", + # Windows + "*-cp38-abi3-win_amd64.whl", + "*-cp38-abi3-win_arm64.whl", + # Free-threading + "*-cp314-cp314t-manylinux2010_x86_64.*.whl", + "*-cp314-cp314t-manylinux2014_aarch64.*.whl", + "*-cp314-cp314t-macosx_10_15_x86_64.whl", + "*-cp314-cp314t-macosx_11_0_arm64.whl", + "*-cp314-cp314t-win_amd64.whl", + "*-cp314-cp314t-win_arm64.whl", +] + + +class Wheel: + def __init__(self, path): + self._path = path + self._name = os.path.basename(path) + + def __repr__(self): + return "<{}(name={}, plat={}, arch={}, pyver={})>".format( + self.__class__.__name__, + self.name, + self.platform(), + self.arch(), + self.pyver(), + ) + + __str__ = __repr__ + + @property + def name(self): + return self._name + + def platform(self): + plat = self.name.split('-')[-1] + pyimpl = self.name.split('-')[3] + ispypy = 'pypy' in pyimpl + if 'linux' in plat: + if ispypy: + return 'pypy_on_linux' + else: + return 'linux' + elif 'win' in plat: + if ispypy: + return 'pypy_on_windows' + else: + return 'windows' + elif 'macosx' in plat: + if ispypy: + return 'pypy_on_macos' + else: + return 'macos' + else: + raise ValueError(f"unknown platform {self.name!r}") + + def arch(self): + if self.name.endswith(('x86_64.whl', 'amd64.whl')): + return '64-bit' + if self.name.endswith(("i686.whl", "win32.whl")): + return '32-bit' + if self.name.endswith("arm64.whl"): + return 'arm64' + if self.name.endswith("aarch64.whl"): + return 'aarch64' + if self.name.endswith("ppc64le.whl"): + return 'ppc64le' + if self.name.endswith("s390x.whl"): + return 's390x' + return '?' + + def pyver(self): + pyver = 'pypy' if self.name.split('-')[3].startswith('pypy') else 'py' + pyver += self.name.split('-')[2][2:] + return pyver + + def size(self): + return os.path.getsize(self._path) + + +class Tarball(Wheel): + def platform(self): + return "source" + + def arch(self): + return "-" + + def pyver(self): + return "-" + + +def check_dist(wheels, tarballs): + """Assert the full expected set of wheels + one sdist is present. + Returns a list of error strings (empty means all good). + """ + names = [w.name for w in wheels] + errors = [ + f"missing wheel: {pat}" + for pat in EXPECTED_WHEELS + if not any(fnmatch.fnmatch(n, pat) for n in names) + ] + errors += [ + f"unexpected wheel: {n}" + for n in names + if not any(fnmatch.fnmatch(n, p) for p in EXPECTED_WHEELS) + ] + if len(tarballs) != 1: + errors.append(f"expected 1 sdist, found {len(tarballs)}") + return errors + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + 'dir', + nargs="?", + default="dist", + help='directory containing tar.gz or wheel files', + ) + parser.add_argument( + "--check", + action="store_true", + help="assert the full expected set of wheels is present", + ) + args = parser.parse_args() + + groups = collections.defaultdict(list) + ls = sorted(os.listdir(args.dir), key=lambda x: x.endswith("tar.gz")) + for name in ls: + path = os.path.join(args.dir, name) + if path.endswith(".whl"): + pkg = Wheel(path) + elif path.endswith(".tar.gz"): + pkg = Tarball(path) + else: + raise ValueError(f"invalid package {path!r}") + groups[pkg.platform()].append(pkg) + + tot_files = 0 + tot_size = 0 + templ = "{:<120} {:>7} {:>8} {:>7}" + for platf, pkgs in groups.items(): + ppn = f"{platf} ({len(pkgs)})" + s = templ.format(ppn, "size", "arch", "pyver") + print_color('\n' + s, color=None, bold=True) + for pkg in sorted(pkgs, key=lambda x: x.name): + tot_files += 1 + tot_size += pkg.size() + s = templ.format( + " " + pkg.name, + bytes2human(pkg.size()), + pkg.arch(), + pkg.pyver(), + ) + if 'pypy' in pkg.pyver(): + print_color(s, color='violet') + else: + print_color(s, color='brown') + + print_color( + f"\n\ntotals: files={tot_files}, size={bytes2human(tot_size)}", + bold=True, + ) + + if args.check: + all_pkgs = [p for pkgs in groups.values() for p in pkgs] + tarballs = [p for p in all_pkgs if isinstance(p, Tarball)] + wheels = [p for p in all_pkgs if not isinstance(p, Tarball)] + errors = check_dist(wheels, tarballs) + if errors: + print_color("\ndist check FAILED:", color='red', bold=True) + for err in errors: + print_color(" " + err, color='red') + print_color( + "\nif intentional, update EXPECTED_WHEELS in " + + os.path.basename(__file__), + bold=True, + ) + sys.exit(1) + print_color("\ndist check: OK", color='green', bold=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/internal/print_downloads.py b/scripts/internal/print_downloads.py new file mode 100755 index 0000000000..1a17650b71 --- /dev/null +++ b/scripts/internal/print_downloads.py @@ -0,0 +1,541 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Print PYPI download statistics. +Useful sites: +* https://pepy.tech/project/psutil +* https://pypistats.org/packages/psutil +* https://hugovk.github.io/top-pypi-packages/. +""" + +import argparse +import collections +import datetime +import functools +import json +import os +import re +import shlex +import subprocess +import sys +import urllib.request + +from psutil._common import bytes2human +from psutil._common import hilite + +AUTH_FILE = os.path.expanduser("~/.pypinfo.json") +CACHE_FILE = os.path.expanduser("~/.cache/psutil-print-downloads.json") +PKGNAME = 'psutil' +DAYS = 30 +CACHE_DAYS = 7 +# pypinfo defaults to 10 rows, which would turn "%" into a share of the +# top 10. Raising it is free: BigQuery scans the same bytes either way. +LIMIT = 1000 +# pypinfo's own --all means "every installer, not just pip". Without it +# we'd miss the ~46% of downloads that come from uv. +PYPINFO = f"pypinfo --json --all --days {DAYS} --limit" +SDIST = "sdist (built from source)" +FREETHREADED = "wheel (free-threaded)" +PYPISTATS_URL = "https://pypistats.org/api/packages/{}/{}" +TOP_PACKAGES_URL = ( + "https://hugovk.dev/top-pypi-packages/top-pypi-packages.min.json" +) +LABEL_WIDTH = 36 +LAST_UPDATE = None +bytes_billed = 0 +MAX_ROWS = 20 +# Python versions that reached end-of-life. +EOL_PYTHONS = {"2.6", "2.7", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9"} + +# CLI args +ALL = False + + +def parse_cli(): + global ALL + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "-a", + "--all", + action="store_true", + help="print more info via BigQuery (expensive)", + ) + args = parser.parse_args() + ALL = args.all + + +def file_cache(fun): + """Cache the decorated function's JSON-serializable return value on + disk for CACHE_DAYS days. + """ + + @functools.wraps(fun) + def wrapper(*args): + key = repr((fun.__name__, args)) + today = datetime.date.today() + try: + with open(CACHE_FILE) as f: + cache = json.load(f) + written = datetime.date.fromisoformat(cache["date"]) + if (today - written).days >= CACHE_DAYS: + raise ValueError("stale") + except (FileNotFoundError, ValueError, KeyError): + cache = {"date": str(today), "entries": {}} + entries = cache.get("entries", {}) + if key in entries: + return entries[key] + ret = fun(*args) + entries[key] = ret + cache["entries"] = entries + os.makedirs(os.path.dirname(CACHE_FILE), exist_ok=True) + with open(CACHE_FILE, "w") as f: + json.dump(cache, f) + return ret + + return wrapper + + +# --- get (free: no credentials, no quota) + + +@file_cache +def pypistats_fetch(kind): + url = PYPISTATS_URL.format(PKGNAME, kind) + with urllib.request.urlopen(url, timeout=30) as resp: + return json.load(resp)["data"] + + +@functools.lru_cache +def pypistats(kind): + """Return a {category: downloads} Counter for the last DAYS days. + Mirror traffic is excluded. + """ + global LAST_UPDATE + rows = pypistats_fetch(kind) + LAST_UPDATE = max(x["date"] for x in rows) + start = datetime.date.fromisoformat(LAST_UPDATE) - datetime.timedelta( + days=DAYS - 1 + ) + totals = collections.Counter() + for row in rows: + if datetime.date.fromisoformat(row["date"]) >= start: + totals[row["category"]] += row["downloads"] + return totals + + +def downloads(): + return sum(pypistats("python_minor").values()) + + +def downloads_pyver(): + return pypistats("python_minor") + + +def downloads_by_system(): + return pypistats("system") + + +@file_cache +def ranking(): + with urllib.request.urlopen(TOP_PACKAGES_URL, timeout=60) as resp: + rows = json.load(resp)["rows"] + for i, row in enumerate(rows, start=1): + if row["project"] == PKGNAME: + return i + raise ValueError(f"can't find {PKGNAME} in {TOP_PACKAGES_URL}") + + +# --- get (BigQuery, --all only) +# +# Everything below costs money. Sizes were measured with `pypinfo -n`. + + +def sh(cmd): + assert os.path.exists(AUTH_FILE) + env = os.environ.copy() + env['GOOGLE_APPLICATION_CREDENTIALS'] = AUTH_FILE + p = subprocess.Popen( + shlex.split(cmd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + env=env, + ) + stdout, stderr = p.communicate() + if p.returncode != 0: + raise RuntimeError(stderr) + assert not stderr, stderr + return stdout.strip() + + +@functools.lru_cache +@file_cache +def query(cmd): + global bytes_billed + import pypinfo # noqa: F401 + + ret = json.loads(sh(cmd)) + bytes_billed += ret['query']['bytes_billed'] + return ret + + +# EXPENSIVE: ~11 GB scanned per call. +def downloads_by_distro(): + return query(f"{PYPINFO} {LIMIT} {PKGNAME} distro") + + +# EXPENSIVE: ~17 GB scanned per call (only once: it's cached). +def downloads_by_system_release(): + """(system, release) rows, shared by the per-OS tables below. + The high limit prevents the many distinct Linux kernel versions + from crowding out the niche OSes. + """ + cmd = f"{PYPINFO} 20000 {PKGNAME} system system-release" + return query(cmd)['rows'] + + +def downloads_by_release(system): + totals = collections.Counter() + for row in downloads_by_system_release(): + if row['system_name'] == system: + totals[row['system_release']] += row['download_count'] + return totals + + +def downloads_by_windows_release(): + # "10" also includes Windows 11: older interpreters report both + # as "10". + return downloads_by_release("Windows") + + +# EXPENSIVE: ~18 GB scanned per call (only once: it's cached). +def downloads_by_dimension(key): + """Aggregate one dimension (version, implementation, + installer_name, cpu, libc_name) out of a single combined query. + BigQuery bills by column, so one query costs less than five. + Counts are near-exact: combos below the row limit are lost. + """ + cmd = f"{PYPINFO} 20000 {PKGNAME} version impl installer cpu libc" + totals = collections.Counter() + for row in query(cmd)['rows']: + totals[row[key]] += row['download_count'] + return totals + + +CPU_ALIASES = { + "": "unknown", + "amd64": "x86_64", + "arm64": "aarch64", + "armv8l": "armv7l", + "i386": "i686", + "i86pc": "x86_64", + "none": "unknown", + "sun4v": "sparc64", + "x86": "i686", +} + +CPU_NAMES = { + "aarch64", + "armv6l", + "armv7l", + "e2k", + "i686", + "loongarch64", + "mips", + "mips64", + "ppc", + "ppc64", + "ppc64le", + "riscv64", + "s390x", + "sparc64", + "sw_64", + "unknown", + "wasm32", + "x86_64", +} + + +def normalize_cpu(name): + name = str(name).lower() + name = CPU_ALIASES.get(name, name) + return name if name in CPU_NAMES else "other" + + +def downloads_by_cpu(): + totals = collections.Counter() + for name, num in downloads_by_dimension('cpu').items(): + totals[normalize_cpu(name)] += num + return totals + + +# platform.release() on macOS returns the Darwin kernel version +DARWIN_TO_MACOS = { + "19": "10.15 Catalina", + "20": "11 Big Sur", + "21": "12 Monterey", + "22": "13 Ventura", + "23": "14 Sonoma", + "24": "15 Sequoia", + "25": "26 Tahoe", +} + + +def downloads_by_macos_release(): + totals = collections.Counter() + for release, num in downloads_by_release("Darwin").items(): + major = release.split(".")[0] + name = DARWIN_TO_MACOS.get(major, f"Darwin {release}") + totals[name] += num + return totals + + +def downloads_by_other_systems(): + """BSD, AIX, SunOS, etc. Excludes Linux, whose system-release is + the kernel version (too many, not interesting). + """ + skip = {"Windows", "Darwin", "Linux", "None", None} + totals = collections.Counter() + for row in downloads_by_system_release(): + if row['system_name'] not in skip: + name = f"{row['system_name']} {row['system_release']}" + totals[name] += row['download_count'] + return totals + + +@file_cache +def bq_monthly_usage_cached(month): + return bq_monthly_usage() + + +def monthly_usage(): + if bytes_billed: + return bq_monthly_usage() + return bq_monthly_usage_cached(datetime.date.today().strftime("%Y-%m")) + + +def bq_monthly_usage(): + """Bytes billed to the BigQuery project since the start of the + month. The free tier is 1 TiB. This query itself bills the 20 MiB + minimum. + """ + from google.cloud import bigquery + + os.environ.setdefault('GOOGLE_APPLICATION_CREDENTIALS', AUTH_FILE) + client = bigquery.Client() + sql = """ + SELECT COALESCE(SUM(total_bytes_billed), 0) AS billed + FROM `region-us`.INFORMATION_SCHEMA.JOBS_BY_USER + WHERE creation_time >= TIMESTAMP(DATE_TRUNC(CURRENT_DATE(), MONTH)) + """ + return next(iter(client.query(sql).result())).billed + + +def downloads_by_wheel(): + """Group downloads by the kind of file fetched. This is the only + way to count free-threaded (no-GIL) usage: those interpreters + report the same version as regular ones, so pyversion can't tell + them apart. Also break sdists down by platform and architecture: + those are the users who would benefit from a new wheel. + """ + # EXPENSIVE: ~45 GB scanned per call, the priciest query here. + # A file gets one row per (system, cpu) combo, hence the big limit: + # cutting the tail undercounts sdist and free-threaded. + cmd = f"{PYPINFO} 20000 {PKGNAME} file system cpu" + totals = collections.Counter() + subs = {SDIST: collections.Counter(), FREETHREADED: collections.Counter()} + for row in query(cmd)['rows']: + name = row['file'] + if name.endswith(".metadata"): + continue # PEP 658 sidecar, not an actual download + num = row['download_count'] + freethreaded = re.search(r"-(cp\d+t)-", name) + if name.endswith((".tar.gz", ".zip")): + totals[SDIST] += num + system = row['system_name'] + if not system or system == "None": + system = "unknown" + subs[SDIST][f"{system} / {normalize_cpu(row['cpu'])}"] += num + elif freethreaded: + totals[FREETHREADED] += num + if normalize_cpu(row['cpu']) == "unknown": + key = "no platform reported by the client" + else: + plat = name.rsplit("-", 1)[-1][: -len(".whl")].split(".")[0] + key = f"{freethreaded.group(1)} / {plat}" + subs[FREETHREADED][key] += num + elif "-abi3-" in name: + totals["wheel (abi3)"] += num + else: + totals["wheel (version specific)"] += num + return totals, subs + + +# --- print + + +def fold(rows, key, limit=MAX_ROWS): + if not limit or len(rows) <= limit: + return rows + tail = rows[limit:] + return rows[:limit] + [{ + key: f"+ {len(tail)} more", + 'download_count': sum(x['download_count'] for x in tail), + }] + + +def print_table(title, left, rows, percent=True, total=None, limit=MAX_ROWS): + if total is None: + total = sum(x['download_count'] for x in rows) + rows = fold(rows, left, limit) + if percent: + header = f"{title:<{LABEL_WIDTH}} {'Downloads':>15} {'%':>7}" + else: + header = f"{title:<{LABEL_WIDTH}} {'Downloads':>15}" + print(hilite(header, color="brown", bold=True)) + print(hilite("-" * len(header), color="grey")) + for row in rows: + num = row['download_count'] + lval = str(row[left] or "null") + line = f"{lval:<{LABEL_WIDTH}} {num:>15,}" + if percent: + line += f" {100 * num / total:>7.2f}" + print(line) + print() + + +def to_rows(totals, key): + """Turn a Counter into the row dicts print_table wants.""" + return [ + {key: name, 'download_count': num} + for name, num in totals.most_common() + ] + + +def print_cheap(): + downs = downloads() + + title = f"psutil downloads in the last {DAYS} days" + print(hilite(title, color="violet", bold=True)) + print(hilite(f"updated at {LAST_UPDATE}", color="grey")) + overall = pypistats("overall") + mirrors = overall["with_mirrors"] - overall["without_mirrors"] + pct = 100 * mirrors / overall["with_mirrors"] + s = f"mirror traffic (excluded from all tables): {pct:.1f}%" + print(hilite(s, color="grey")) + print() + + data = [ + {'what': 'Per month', 'download_count': downs}, + {'what': 'Per day', 'download_count': int(downs / DAYS)}, + {'what': 'PYPI ranking', 'download_count': ranking()}, + ] + print_table('Overview', 'what', data, percent=False) + print_table( + 'Operating systems', + 'system_name', + to_rows(downloads_by_system(), 'system_name'), + ) + print_table( + 'Python versions', + 'python_version', + to_rows(downloads_pyver(), 'python_version'), + ) + eol = sum( + num for ver, num in downloads_pyver().items() if ver in EOL_PYTHONS + ) + s = f"downloads from EOL Pythons: {eol:,} ({100 * eol / downs:.1f}%)" + print(hilite(s, color="grey")) + + +def print_expensive(): + print_table( + 'psutil versions', + 'version', + to_rows(downloads_by_dimension('version'), 'version'), + ) + wheels, subs = downloads_by_wheel() + rows = [] + for row in to_rows(wheels, 'wheel_type'): + rows.append(row) + if row['wheel_type'] == SDIST: + rows.extend( + { + 'wheel_type': " " + r['wheel_type'], + 'download_count': r['download_count'], + } + for r in fold(to_rows(subs[SDIST], 'wheel_type'), 'wheel_type') + ) + print_table( + 'Wheel types', + 'wheel_type', + rows, + total=sum(wheels.values()), + limit=None, + ) + print_table( + 'Free-threaded wheels', + 'wheel', + to_rows(subs[FREETHREADED], 'wheel'), + ) + print_table( + 'Implementations', + 'implementation', + to_rows(downloads_by_dimension('implementation'), 'implementation'), + ) + print_table( + 'Installers', + 'installer_name', + to_rows(downloads_by_dimension('installer_name'), 'installer_name'), + ) + print_table('CPUs', 'cpu', to_rows(downloads_by_cpu(), 'cpu')) + print_table( + 'libc', + 'libc_name', + to_rows(downloads_by_dimension('libc_name'), 'libc_name'), + ) + print_table( + 'Windows versions', + 'windows_release', + to_rows(downloads_by_windows_release(), 'windows_release'), + ) + print_table( + 'macOS versions', + 'macos_release', + to_rows(downloads_by_macos_release(), 'macos_release'), + ) + print_table( + 'Other systems', + 'system_release', + to_rows(downloads_by_other_systems(), 'system_release'), + ) + print_table('Distros', 'distro_name', downloads_by_distro()['rows']) + + billed = monthly_usage() + pct = 100 * billed / 1024**4 + s = ( + f"BigQuery free tier used this month: {bytes2human(billed)} of 1" + f" TiB ({pct:.0f}%)" + ) + print(hilite(s, color="grey")) + + +def main(): + parse_cli() + print_cheap() + if ALL: + print_expensive() + + +if __name__ == '__main__': + try: + main() + finally: + if bytes_billed: + print(f"bytes billed: {bytes_billed}", file=sys.stderr) diff --git a/scripts/internal/print_hashes.py b/scripts/internal/print_hashes.py new file mode 100755 index 0000000000..1161268d46 --- /dev/null +++ b/scripts/internal/print_hashes.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Prints files hashes, see: +https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode. +""" + +import argparse +import hashlib +import os + + +def csum(file, kind): + h = hashlib.new(kind) + with open(file, "rb") as f: + h.update(f.read()) + return h.hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "dir", + type=str, + nargs="?", + help="directory containing tar.gz or wheel files", + default="dist/", + ) + args = parser.parse_args() + for name in sorted(os.listdir(args.dir)): + file = os.path.join(args.dir, name) + if os.path.isfile(file): + md5 = csum(file, "md5") + sha256 = csum(file, "sha256") + print(f"{os.path.basename(file)}\nmd5: {md5}\nsha256: {sha256}\n") + else: + print(f"skipping {file!r} (not a file)") + + +if __name__ == "__main__": + main() diff --git a/scripts/internal/print_sysinfo.py b/scripts/internal/print_sysinfo.py new file mode 100755 index 0000000000..1f872f73a4 --- /dev/null +++ b/scripts/internal/print_sysinfo.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Print system information. Run before CI test run.""" + +import datetime +import getpass +import locale +import os +import pathlib +import platform +import shlex +import shutil +import subprocess +import sys + +import psutil +from psutil import bytes2human + +try: + import pip +except ImportError: + pip = None +try: + import wheel +except ImportError: + wheel = None + + +ROOT_DIR = pathlib.Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(ROOT_DIR)) +from _bootstrap import load_module # noqa: E402 + + +def sh(cmd): + if isinstance(cmd, str): + cmd = shlex.split(cmd) + return subprocess.check_output(cmd, universal_newlines=True).strip() + + +tests_init = ROOT_DIR / "tests" / "__init__.py" + +tests_init_mod = load_module(tests_init) + + +def main(): + info = {} + + # python + info['python'] = ', '.join([ + platform.python_implementation(), + platform.python_version(), + platform.python_compiler(), + ]) + + # OS + if psutil.LINUX and shutil.which("lsb_release"): + info['OS'] = sh('lsb_release -d -s') + elif psutil.OSX: + info['OS'] = f"Darwin {platform.mac_ver()[0]}" + elif psutil.WINDOWS: + info['OS'] = "Windows " + ' '.join(map(str, platform.win32_ver())) + if hasattr(platform, 'win32_edition'): + info['OS'] += ", " + platform.win32_edition() + else: + info['OS'] = f"{platform.system()} {platform.version()}" + info['arch'] = ', '.join( + list(platform.architecture()) + [platform.machine()] + ) + if psutil.POSIX: + info['kernel'] = platform.uname()[2] + + # pip + info['pip'] = getattr(pip, '__version__', 'not installed') + if wheel is not None: + info['pip'] += f" (wheel={wheel.__version__})" + + # UNIX + if psutil.POSIX: + if shutil.which("gcc"): + out = sh(['gcc', '--version']) + info['gcc'] = str(out).split('\n')[0] + else: + info['gcc'] = 'not installed' + s = platform.libc_ver()[1] + if s: + info['glibc'] = s + + # system + info['fs-encoding'] = sys.getfilesystemencoding() + lang = locale.getlocale() + info['lang'] = f"{lang[0]}, {lang[1]}" + info['boot-time'] = datetime.datetime.fromtimestamp( + psutil.boot_time() + ).strftime("%Y-%m-%d %H:%M:%S") + info['time'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + info['user'] = getpass.getuser() + info['home'] = os.path.expanduser("~") + info['cwd'] = os.getcwd() + info['pyexe'] = tests_init_mod.PYTHON_EXE + info['hostname'] = platform.node() + info['PID'] = os.getpid() + + # metrics + info['cpus'] = psutil.cpu_count() + loadavg = tuple(x / psutil.cpu_count() * 100 for x in psutil.getloadavg()) + info['loadavg'] = ( + f"{loadavg[0]:.1f}%, {loadavg[1]:.1f}%, {loadavg[2]:.1f}%" + ) + mem = psutil.virtual_memory() + info['memory'] = "{}%%, used={}, total={}".format( + int(mem.percent), + bytes2human(mem.used), + bytes2human(mem.total), + ) + swap = psutil.swap_memory() + info['swap'] = "{}%%, used={}, total={}".format( + int(swap.percent), + bytes2human(swap.used), + bytes2human(swap.total), + ) + + # constants + constants = sorted([ + x + for x in dir(tests_init_mod) + if x.isupper() and getattr(tests_init_mod, x) is True + ]) + info['constants'] = "\n ".join(constants) + + # processes + # info['pids'] = len(psutil.pids()) + # pinfo = psutil.Process().as_dict() + # pinfo.pop('memory_maps', None) + # pinfo["environ"] = {k: os.environ[k] for k in sorted(os.environ)} + # info['proc'] = pprint.pformat(pinfo) + + # print + print("=" * 70) + for k, v in info.items(): + print("{:<17} {}".format(k + ":", v)) + print("=" * 70) + sys.stdout.flush() + + +if __name__ == "__main__": + main() diff --git a/scripts/internal/purge_installation.py b/scripts/internal/purge_installation.py new file mode 100755 index 0000000000..94c5f287d6 --- /dev/null +++ b/scripts/internal/purge_installation.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Purge psutil installation by removing psutil-related files and +directories found in site-packages directories. This is needed mainly +because sometimes "import psutil" imports a leftover installation +from site-packages directory instead of the main working directory. +""" + +import os +import shutil +import site + +PKGNAME = "psutil" + +locations = [site.getusersitepackages()] + site.getsitepackages() + + +def rmpath(path): + if os.path.isdir(path): + print("rmdir " + path) + shutil.rmtree(path) + else: + print("rm " + path) + os.remove(path) + + +def purge(): + for root in locations: + if os.path.isdir(root): + for name in os.listdir(root): + if PKGNAME in name: + abspath = os.path.join(root, name) + rmpath(abspath) + + +def purge_windows(): + r"""Uninstalling psutil on Windows is more tricky. On "import + psutil" tests may import a psutil version living in + C:\PythonXY\Lib\site-packages which is not what we want, so other + than "pip uninstall psutil" we also manually remove stuff from + site-packages dirs. + """ + for dir in locations: + for name in os.listdir(dir): + path = os.path.join(dir, name) + if name.startswith(PKGNAME): + rmpath(path) + elif name == 'easy-install.pth': + # easy_install can add a line (installation path) into + # easy-install.pth; that line alters sys.path. + path = os.path.join(dir, name) + with open(path) as f: + lines = f.readlines() + hasit = False + for line in lines: + if PKGNAME in line: + hasit = True + break + if hasit: + with open(path, "w") as f: + for line in lines: + if PKGNAME not in line: + f.write(line) + else: + print(f"removed line {line!r} from {path!r}") + + +def main(): + purge() + if os.name == "nt": + purge_windows() + + +if __name__ == "__main__": + main() diff --git a/scripts/iotop.py b/scripts/iotop.py index 200926ed85..a0ed73a952 100755 --- a/scripts/iotop.py +++ b/scripts/iotop.py @@ -1,20 +1,19 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -A clone of iotop (http://guichaz.free.fr/iotop/) showing real time +"""A clone of iotop (http://guichaz.free.fr/iotop/) showing real time disk I/O statistics. -It works on Linux only (FreeBSD and OSX are missing support for IO +It works on Linux only (FreeBSD and macOS are missing support for IO counters). It doesn't work on Windows as curses module is required. Example output: -$ python scripts/iotop.py +$ python3 scripts/iotop.py Total DISK READ: 0.00 B/s | Total DISK WRITE: 472.00 K/s PID USER DISK READ DISK WRITE COMMAND 13155 giampao 0.00 B/s 428.00 K/s /usr/bin/google-chrome-beta @@ -30,31 +29,22 @@ Author: Giampaolo Rodola' """ -import atexit -import time import sys +import time + try: import curses except ImportError: sys.exit('platform not supported') import psutil - - -# --- curses stuff -def tear_down(): - win.keypad(0) - curses.nocbreak() - curses.echo() - curses.endwin() +from psutil import bytes2human win = curses.initscr() -atexit.register(tear_down) -curses.endwin() lineno = 0 -def print_line(line, highlight=False): +def printl(line, highlight=False): """A thin wrapper around curses's addstr().""" global lineno try: @@ -69,36 +59,17 @@ def print_line(line, highlight=False): raise else: lineno += 1 -# --- /curses stuff - - -def bytes2human(n): - """ - >>> bytes2human(10000) - '9.8 K/s' - >>> bytes2human(100001221) - '95.4 M/s' - """ - symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') - prefix = {} - for i, s in enumerate(symbols): - prefix[s] = 1 << (i + 1) * 10 - for s in reversed(symbols): - if n >= prefix[s]: - value = float(n) / prefix[s] - return '%.2f %s/s' % (value, s) - return '%.2f B/s' % (n) def poll(interval): - """Calculate IO usage by comparing IO statics before and + """Calculate IO usage by comparing IO statistics before and after the interval. Return a tuple including all currently running processes sorted by IO activity and total disks I/O activity. """ # first get a list of all processes and disk io counters - procs = [p for p in psutil.process_iter()] - for p in procs[:]: + procs = list(psutil.process_iter()) + for p in procs.copy(): try: p._before = p.io_counters() except psutil.Error: @@ -110,15 +81,16 @@ def poll(interval): time.sleep(interval) # then retrieve the same info again - for p in procs[:]: - try: - p._after = p.io_counters() - p._cmdline = ' '.join(p.cmdline()) - if not p._cmdline: - p._cmdline = p.name() - p._username = p.username() - except (psutil.NoSuchProcess, psutil.ZombieProcess): - procs.remove(p) + for p in procs.copy(): + with p.oneshot(): + try: + p._after = p.io_counters() + p._cmdline = ' '.join(p.cmdline()) + if not p._cmdline: + p._cmdline = p.name() + p._username = p.username() + except (psutil.NoSuchProcess, psutil.ZombieProcess): + procs.remove(p) disks_after = psutil.disk_io_counters() # finally calculate results by comparing data before and @@ -141,39 +113,67 @@ def poll(interval): def refresh_window(procs, disks_read, disks_write): """Print results on screen by using curses.""" curses.endwin() - templ = "%-5s %-7s %11s %11s %s" + templ = "{:<5} {:<7} {:>11} {:>11} {}" win.erase() - disks_tot = "Total DISK READ: %s | Total DISK WRITE: %s" \ - % (bytes2human(disks_read), bytes2human(disks_write)) - print_line(disks_tot) + disks_tot = "Total DISK READ: {} | Total DISK WRITE: {}".format( + bytes2human(disks_read), + bytes2human(disks_write), + ) + printl(disks_tot) - header = templ % ("PID", "USER", "DISK READ", "DISK WRITE", "COMMAND") - print_line(header, highlight=True) + header = templ.format("PID", "USER", "DISK READ", "DISK WRITE", "COMMAND") + printl(header, highlight=True) for p in procs: - line = templ % ( + line = templ.format( p.pid, p._username[:7], bytes2human(p._read_per_sec), bytes2human(p._write_per_sec), - p._cmdline) + p._cmdline, + ) try: - print_line(line) + printl(line) except curses.error: break win.refresh() +def setup(): + curses.start_color() + curses.use_default_colors() + for i in range(curses.COLORS): + curses.init_pair(i + 1, i, -1) + curses.endwin() + win.nodelay(1) + + +def tear_down(): + win.keypad(0) + curses.nocbreak() + curses.echo() + curses.endwin() + + def main(): + global lineno + setup() try: interval = 0 while True: + if win.getch() == ord('q'): + break args = poll(interval) refresh_window(*args) - interval = 1 + lineno = 0 + interval = 0.5 + time.sleep(interval) except (KeyboardInterrupt, SystemExit): pass + finally: + tear_down() + if __name__ == '__main__': main() diff --git a/scripts/killall.py b/scripts/killall.py index b548e7bc57..532e8b15ce 100755 --- a/scripts/killall.py +++ b/scripts/killall.py @@ -1,32 +1,33 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -Kill a process by name. -""" +"""Kill a process by name.""" import os import sys + import psutil def main(): if len(sys.argv) != 2: - sys.exit('usage: %s name' % __file__) + sys.exit(f"usage: {__file__} name") else: - NAME = sys.argv[1] + name = sys.argv[1] killed = [] for proc in psutil.process_iter(): - if proc.name() == NAME and proc.pid != os.getpid(): + if proc.name() == name and proc.pid != os.getpid(): proc.kill() killed.append(proc.pid) if not killed: - sys.exit('%s: no process found' % NAME) + sys.exit(f"{name}: no process found") else: sys.exit(0) -sys.exit(main()) + +if __name__ == '__main__': + main() diff --git a/scripts/meminfo.py b/scripts/meminfo.py index 3546960b2a..9a26aed5a4 100755 --- a/scripts/meminfo.py +++ b/scripts/meminfo.py @@ -1,13 +1,12 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -Print system memory information. +"""Print system memory information. -$ python scripts/meminfo.py +$ python3 scripts/meminfo.py MEMORY ------ Total : 9.7G @@ -31,23 +30,7 @@ """ import psutil - - -def bytes2human(n): - # http://code.activestate.com/recipes/578019 - # >>> bytes2human(10000) - # '9.8K' - # >>> bytes2human(100001221) - # '95.4M' - symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') - prefix = {} - for i, s in enumerate(symbols): - prefix[s] = 1 << (i + 1) * 10 - for s in reversed(symbols): - if n >= prefix[s]: - value = float(n) / prefix[s] - return '%.1f%s' % (value, s) - return "%sB" % n +from psutil import bytes2human def pprint_ntuple(nt): @@ -55,7 +38,7 @@ def pprint_ntuple(nt): value = getattr(nt, name) if name != 'percent': value = bytes2human(value) - print('%-10s : %7s' % (name.capitalize(), value)) + print('{:<10} : {:>7}'.format(name.capitalize(), value)) def main(): @@ -64,5 +47,6 @@ def main(): print('\nSWAP\n----') pprint_ntuple(psutil.swap_memory()) + if __name__ == '__main__': main() diff --git a/scripts/netstat.py b/scripts/netstat.py index a5e171bc49..9f26ee0790 100755 --- a/scripts/netstat.py +++ b/scripts/netstat.py @@ -1,13 +1,12 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -A clone of 'netstat -antp' on Linux. +"""A clone of 'netstat -antp' on Linux. -$ python scripts/netstat.py +$ python3 scripts/netstat.py Proto Local address Remote address Status PID Program name tcp 127.0.0.1:48256 127.0.0.1:45884 ESTABLISHED 13646 chrome tcp 127.0.0.1:47073 127.0.0.1:45884 ESTABLISHED 13646 chrome @@ -20,11 +19,12 @@ """ import socket -from socket import AF_INET, SOCK_STREAM, SOCK_DGRAM +from socket import AF_INET +from socket import SOCK_DGRAM +from socket import SOCK_STREAM import psutil - AD = "-" AF_INET6 = getattr(socket, 'AF_INET6', object()) proto_map = { @@ -36,29 +36,35 @@ def main(): - templ = "%-5s %-30s %-30s %-13s %-6s %s" - print(templ % ( - "Proto", "Local address", "Remote address", "Status", "PID", - "Program name")) + templ = "{:<5} {:<30} {:<30} {:<13} {:<6} {}" + header = templ.format( + "Proto", + "Local address", + "Remote address", + "Status", + "PID", + "Program name", + ) + print(header) proc_names = {} - for p in psutil.process_iter(): - try: - proc_names[p.pid] = p.name() - except psutil.Error: - pass + for p in psutil.process_iter(['pid', 'name']): + proc_names[p.pid] = p.name() for c in psutil.net_connections(kind='inet'): - laddr = "%s:%s" % (c.laddr) + laddr = f"{c.laddr[0]}:{c.laddr[1]}" raddr = "" if c.raddr: - raddr = "%s:%s" % (c.raddr) - print(templ % ( + raddr = f"{c.raddr[0]}:{c.raddr[1]}" + name = proc_names.get(c.pid, '?') or '' + line = templ.format( proto_map[(c.family, c.type)], laddr, raddr or AD, c.status, c.pid or AD, - proc_names.get(c.pid, '?')[:15], - )) + name[:15], + ) + print(line) + if __name__ == '__main__': main() diff --git a/scripts/nettop.py b/scripts/nettop.py index acfa650066..98732b5ff2 100755 --- a/scripts/nettop.py +++ b/scripts/nettop.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # $Id: iotop.py 1160 2011-10-14 18:50:36Z g.rodola@gmail.com $ # @@ -6,12 +6,11 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -Shows real-time network statistics. +"""Shows real-time network statistics. Author: Giampaolo Rodola' -$ python scripts/nettop.py +$ python3 scripts/nettop.py ----------------------------------------------------------- total bytes: sent: 1.49 G received: 4.82 G total packets: sent: 7338724 received: 8082712 @@ -31,31 +30,22 @@ pkts-recv 1214470 0 """ -import atexit -import time import sys +import time + try: import curses except ImportError: sys.exit('platform not supported') import psutil +from psutil import bytes2human - -# --- curses stuff -def tear_down(): - win.keypad(0) - curses.nocbreak() - curses.echo() - curses.endwin() - -win = curses.initscr() -atexit.register(tear_down) -curses.endwin() lineno = 0 +win = curses.initscr() -def print_line(line, highlight=False): +def printl(line, highlight=False): """A thin wrapper around curses's addstr().""" global lineno try: @@ -70,25 +60,6 @@ def print_line(line, highlight=False): raise else: lineno += 1 -# --- curses stuff - - -def bytes2human(n): - """ - >>> bytes2human(10000) - '9.8 K' - >>> bytes2human(100001221) - '95.4 M' - """ - symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') - prefix = {} - for i, s in enumerate(symbols): - prefix[s] = 1 << (i + 1) * 10 - for s in reversed(symbols): - if n >= prefix[s]: - value = float(n) / prefix[s] - return '%.2f %s' % (value, s) - return '%.2f B' % (n) def poll(interval): @@ -107,59 +78,83 @@ def refresh_window(tot_before, tot_after, pnic_before, pnic_after): global lineno # totals - print_line("total bytes: sent: %-10s received: %s" % ( - bytes2human(tot_after.bytes_sent), - bytes2human(tot_after.bytes_recv)) + printl( + "total bytes: sent: {:<10} received: {}".format( + bytes2human(tot_after.bytes_sent), + bytes2human(tot_after.bytes_recv), + ) ) - print_line("total packets: sent: %-10s received: %s" % ( - tot_after.packets_sent, tot_after.packets_recv)) # per-network interface details: let's sort network interfaces so # that the ones which generated more traffic are shown first - print_line("") + printl("") nic_names = list(pnic_after.keys()) nic_names.sort(key=lambda x: sum(pnic_after[x]), reverse=True) for name in nic_names: stats_before = pnic_before[name] stats_after = pnic_after[name] - templ = "%-15s %15s %15s" - print_line(templ % (name, "TOTAL", "PER-SEC"), highlight=True) - print_line(templ % ( + templ = "{:<15s} {:>15} {:>15}" + # fmt: off + printl(templ.format(name, "TOTAL", "PER-SEC"), highlight=True) + printl(templ.format( "bytes-sent", bytes2human(stats_after.bytes_sent), bytes2human( stats_after.bytes_sent - stats_before.bytes_sent) + '/s', )) - print_line(templ % ( + printl(templ.format( "bytes-recv", bytes2human(stats_after.bytes_recv), bytes2human( stats_after.bytes_recv - stats_before.bytes_recv) + '/s', )) - print_line(templ % ( + printl(templ.format( "pkts-sent", stats_after.packets_sent, stats_after.packets_sent - stats_before.packets_sent, )) - print_line(templ % ( + printl(templ.format( "pkts-recv", stats_after.packets_recv, stats_after.packets_recv - stats_before.packets_recv, )) - print_line("") + printl("") + # fmt: on win.refresh() lineno = 0 +def setup(): + curses.start_color() + curses.use_default_colors() + for i in range(curses.COLORS): + curses.init_pair(i + 1, i, -1) + curses.endwin() + win.nodelay(1) + + +def tear_down(): + win.keypad(0) + curses.nocbreak() + curses.echo() + curses.endwin() + + def main(): + setup() try: interval = 0 while True: + if win.getch() == ord('q'): + break args = poll(interval) refresh_window(*args) - interval = 1 + interval = 0.5 except (KeyboardInterrupt, SystemExit): pass + finally: + tear_down() + if __name__ == '__main__': main() diff --git a/scripts/pidof.py b/scripts/pidof.py index 8692a3152b..2353126621 100755 --- a/scripts/pidof.py +++ b/scripts/pidof.py @@ -1,53 +1,40 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola', karthikrev. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -A clone of 'pidof' cmdline utility. +"""A clone of 'pidof' cmdline utility. + $ pidof python 1140 1138 1136 1134 1133 1129 1127 1125 1121 1120 1119 """ -from __future__ import print_function -import psutil import sys +import psutil + def pidof(pgname): - pids = [] - for proc in psutil.process_iter(): - # search for matches in the process name and cmdline - try: - name = proc.name() - except psutil.Error: - pass - else: - if name == pgname: - pids.append(str(proc.pid)) - continue - - try: - cmdline = proc.cmdline() - except psutil.Error: - pass - else: - if cmdline and cmdline[0] == pgname: - pids.append(str(proc.pid)) - - return pids + # search for matches in the process name and cmdline + return [ + str(proc.pid) + for proc in psutil.process_iter(['name', 'cmdline']) + if proc.name() == pgname + or (proc.cmdline() and proc.cmdline()[0] == pgname) + ] def main(): if len(sys.argv) != 2: - sys.exit('usage: %s pgname' % __file__) + sys.exit(f"usage: {__file__} pgname") else: pgname = sys.argv[1] pids = pidof(pgname) if pids: print(" ".join(pids)) + if __name__ == '__main__': main() diff --git a/scripts/pmap.py b/scripts/pmap.py index f0d53355fe..96ed73a9f5 100755 --- a/scripts/pmap.py +++ b/scripts/pmap.py @@ -1,19 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -A clone of 'pmap' utility on Linux, 'vmmap' on OSX and 'procstat -v' on BSD. -Report memory map of a process. +"""A clone of 'pmap' utility on Linux, 'vmmap' on macOS and 'procstat +-v' on BSD. Report memory map of a process. -$ python scripts/pmap.py 32402 -pid=32402, name=hg +$ python3 scripts/pmap.py 32402 Address RSS Mode Mapping -0000000000400000 1200K r-xp /usr/bin/python2.7 -0000000000838000 4K r--p /usr/bin/python2.7 -0000000000839000 304K rw-p /usr/bin/python2.7 +0000000000400000 1200K r-xp /usr/bin/python3.7 +0000000000838000 4K r--p /usr/bin/python3.7 +0000000000839000 304K rw-p /usr/bin/python3.7 00000000008ae000 68K rw-p [anon] 000000000275e000 5396K rw-p [heap] 00002b29bb1e0000 124K r-xp /lib/x86_64-linux-gnu/ld-2.17.so @@ -30,28 +28,41 @@ ... """ +import shutil import sys import psutil +from psutil import bytes2human + + +def safe_print(s): + s = s[: shutil.get_terminal_size()[0]] + try: + print(s) + except UnicodeEncodeError: + print(s.encode('ascii', 'ignore').decode()) def main(): if len(sys.argv) != 2: sys.exit('usage: pmap ') p = psutil.Process(int(sys.argv[1])) - print("pid=%s, name=%s" % (p.pid, p.name())) - templ = "%-16s %10s %-7s %s" - print(templ % ("Address", "RSS", "Mode", "Mapping")) + templ = "{:<20} {:>10} {:<7} {}" + print(templ.format("Address", "RSS", "Mode", "Mapping")) total_rss = 0 for m in p.memory_maps(grouped=False): total_rss += m.rss - print(templ % ( + line = templ.format( m.addr.split('-')[0].zfill(16), - str(m.rss / 1024) + 'K', + bytes2human(m.rss), m.perms, - m.path)) - print("-" * 33) - print(templ % ("Total", str(total_rss / 1024) + 'K', '', '')) + m.path, + ) + safe_print(line) + print("-" * 31) + print(templ.format("Total", bytes2human(total_rss), "", "")) + safe_print(f"PID = {p.pid}, name = {p.name()}") + if __name__ == '__main__': main() diff --git a/scripts/procinfo.py b/scripts/procinfo.py index 9990086f10..0a4125417f 100755 --- a/scripts/procinfo.py +++ b/scripts/procinfo.py @@ -1,141 +1,252 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -Print detailed information about a process. +"""Print detailed information about a process. + Author: Giampaolo Rodola' -$ python scripts/process_detail.py -pid 820 -name python -exe /usr/bin/python2.7 -parent 29613 (bash) -cmdline python scripts/process_detail.py -started 2014-41-27 03:41 -user giampaolo -uids real=1000, effective=1000, saved=1000 -gids real=1000, effective=1000, saved=1000 -terminal /dev/pts/17 -cwd /ssd/svn/psutil -memory 0.1% (resident=10.6M, virtual=58.5M) -cpu 0.0% (user=0.09, system=0.0) -status running -niceness 0 -num threads 1 -I/O bytes-read=0B, bytes-written=0B -open files -running threads id=820, user-time=0.09, sys-time=0.0 +$ python3 scripts/procinfo.py +pid 4600 +name chrome +parent 4554 (bash) +exe /opt/google/chrome/chrome +cwd /home/giampaolo +cmdline /opt/google/chrome/chrome +started 2016-09-19 11:12 +cpu-tspent 27:27.68 +cpu-times user=8914.32, system=3530.59, + children_user=1.46, children_system=1.31 +cpu-affinity [0, 1, 2, 3, 4, 5, 6, 7] +memory rss=520.5M, vms=1.9G, shared=132.6M, text=95.0M, lib=0B, + data=816.5M, dirty=0B +memory % 3.26 +user giampaolo +uids real=1000, effective=1000, saved=1000 +uids real=1000, effective=1000, saved=1000 +terminal /dev/pts/2 +status sleeping +nice 0 +ionice class=IOPriority.IOPRIO_CLASS_NONE, value=0 +num-threads 47 +num-fds 379 +I/O read_count=96.6M, write_count=80.7M, + read_bytes=293.2M, write_bytes=24.5G +ctx-switches voluntary=30426463, involuntary=460108 +children PID NAME + 4605 cat + 4606 cat + 4609 chrome + 4669 chrome +open-files PATH + /opt/google/chrome/icudtl.dat + /opt/google/chrome/snapshot_blob.bin + /opt/google/chrome/natives_blob.bin + /opt/google/chrome/chrome_100_percent.pak + [...] +connections PROTO LOCAL ADDR REMOTE ADDR STATUS + UDP 10.0.0.3:3693 *:* NONE + TCP 10.0.0.3:55102 172.217.22.14:443 ESTABLISHED + UDP 10.0.0.3:35172 *:* NONE + TCP 10.0.0.3:32922 172.217.16.163:443 ESTABLISHED + UDP :::5353 *:* NONE + UDP 10.0.0.3:59925 *:* NONE +threads TID USER SYSTEM + 11795 0.7 1.35 + 11796 0.68 1.37 + 15887 0.74 0.03 + 19055 0.77 0.01 + [...] + total=47 +res-limits RLIMIT SOFT HARD + virtualmem infinity infinity + coredumpsize 0 infinity + cputime infinity infinity + datasize infinity infinity + filesize infinity infinity + locks infinity infinity + memlock 65536 65536 + msgqueue 819200 819200 + nice 0 0 + openfiles 8192 65536 + maxprocesses 63304 63304 + rss infinity infinity + realtimeprio 0 0 + rtimesched infinity infinity + sigspending 63304 63304 + stack 8388608 infinity +mem-maps RSS PATH + 381.4M [anon] + 62.8M /opt/google/chrome/chrome + 15.8M /home/giampaolo/.config/google-chrome/Default/History + 6.6M /home/giampaolo/.config/google-chrome/Default/Favicons + [...] """ +import argparse import datetime -import os import socket import sys import psutil +from psutil import bytes2human - -POSIX = os.name == 'posix' - - -def convert_bytes(n): - symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') - prefix = {} - for i, s in enumerate(symbols): - prefix[s] = 1 << (i + 1) * 10 - for s in reversed(symbols): - if n >= prefix[s]: - value = float(n) / prefix[s] - return '%.1f%s' % (value, s) - return "%sB" % n +ACCESS_DENIED = '' +NON_VERBOSE_ITERATIONS = 4 +RLIMITS_MAP = { + "RLIMIT_AS": "virtualmem", + "RLIMIT_CORE": "coredumpsize", + "RLIMIT_CPU": "cputime", + "RLIMIT_DATA": "datasize", + "RLIMIT_FSIZE": "filesize", + "RLIMIT_MEMLOCK": "memlock", + "RLIMIT_MSGQUEUE": "msgqueue", + "RLIMIT_NICE": "nice", + "RLIMIT_NOFILE": "openfiles", + "RLIMIT_NPROC": "maxprocesses", + "RLIMIT_NPTS": "pseudoterms", + "RLIMIT_RSS": "rss", + "RLIMIT_RTPRIO": "realtimeprio", + "RLIMIT_RTTIME": "rtimesched", + "RLIMIT_SBSIZE": "sockbufsize", + "RLIMIT_SIGPENDING": "sigspending", + "RLIMIT_STACK": "stack", + "RLIMIT_SWAP": "swapuse", +} def print_(a, b): - if sys.stdout.isatty() and POSIX: - fmt = '\x1b[1;32m%-17s\x1b[0m %s' % (a, b) + if sys.stdout.isatty() and psutil.POSIX: + fmt = "\x1b[1;32m{:<13}\x1b[0m {}".format(a, b) else: - fmt = '%-15s %s' % (a, b) + fmt = "{:<11} {}".format(a, b) print(fmt) -def run(pid): - ACCESS_DENIED = '' +def str_ntuple(nt, convert_bytes=False): + if nt == ACCESS_DENIED: + return "" + if not convert_bytes: + return ", ".join([f"{x}={getattr(nt, x)}" for x in nt._fields]) + else: + return ", ".join( + [f"{x}={bytes2human(getattr(nt, x))}" for x in nt._fields] + ) + + +def run(pid, verbose=False): try: - p = psutil.Process(pid) - pinfo = p.as_dict(ad_value=ACCESS_DENIED) + proc = psutil.Process(pid) + pinfo = proc.as_dict(ad_value=ACCESS_DENIED) except psutil.NoSuchProcess as err: sys.exit(str(err)) - try: - parent = p.parent() - if parent: - parent = '(%s)' % parent.name() - else: + # collect other proc info + with proc.oneshot(): + try: + parent = proc.parent() + parent = f"({parent.name()})" if parent else "" + except psutil.Error: parent = '' - except psutil.Error: - parent = '' - if pinfo['create_time'] != ACCESS_DENIED: - started = datetime.datetime.fromtimestamp( - pinfo['create_time']).strftime('%Y-%m-%d %H:%M') - else: - started = ACCESS_DENIED - io = pinfo.get('io_counters', ACCESS_DENIED) - if pinfo['memory_info'] != ACCESS_DENIED: - mem = '%s%% (resident=%s, virtual=%s) ' % ( - round(pinfo['memory_percent'], 1), - convert_bytes(pinfo['memory_info'].rss), - convert_bytes(pinfo['memory_info'].vms)) - else: - mem = ACCESS_DENIED - children = p.children() + try: + pinfo['children'] = proc.children() + except psutil.Error: + pinfo['children'] = [] + if pinfo['create_time']: + started = datetime.datetime.fromtimestamp( + pinfo['create_time'] + ).strftime('%Y-%m-%d %H:%M') + else: + started = ACCESS_DENIED + # here we go print_('pid', pinfo['pid']) print_('name', pinfo['name']) + print_('parent', f"{pinfo['ppid']} {parent}") print_('exe', pinfo['exe']) - print_('parent', '%s %s' % (pinfo['ppid'], parent)) + print_('cwd', pinfo['cwd']) print_('cmdline', ' '.join(pinfo['cmdline'])) print_('started', started) + + cpu_tot_time = datetime.timedelta(seconds=sum(pinfo['cpu_times'])) + cpu_tot_time = "{}:{}.{}".format( + cpu_tot_time.seconds // 60 % 60, + str(cpu_tot_time.seconds % 60).zfill(2), + str(cpu_tot_time.microseconds)[:2], + ) + print_('cpu-tspent', cpu_tot_time) + print_('cpu-times', str_ntuple(pinfo['cpu_times'])) + if hasattr(proc, "cpu_affinity"): + print_("cpu-affinity", pinfo["cpu_affinity"]) + if hasattr(proc, "cpu_num"): + print_("cpu-num", pinfo["cpu_num"]) + + print_('memory', str_ntuple(pinfo['memory_info'], convert_bytes=True)) + print_('memory %', round(pinfo['memory_percent'], 2)) print_('user', pinfo['username']) - if POSIX and pinfo['uids'] and pinfo['gids']: - print_('uids', 'real=%s, effective=%s, saved=%s' % pinfo['uids']) - if POSIX and pinfo['gids']: - print_('gids', 'real=%s, effective=%s, saved=%s' % pinfo['gids']) - if POSIX: + if psutil.POSIX: + print_('uids', str_ntuple(pinfo['uids'])) + if psutil.POSIX: + print_('uids', str_ntuple(pinfo['uids'])) + if psutil.POSIX: print_('terminal', pinfo['terminal'] or '') - print_('cwd', pinfo['cwd']) - print_('memory', mem) - print_('cpu', '%s%% (user=%s, system=%s)' % ( - pinfo['cpu_percent'], - getattr(pinfo['cpu_times'], 'user', '?'), - getattr(pinfo['cpu_times'], 'system', '?'))) + print_('status', pinfo['status']) - print_('niceness', pinfo['nice']) - print_('num threads', pinfo['num_threads']) - if io != ACCESS_DENIED: - print_('I/O', 'bytes-read=%s, bytes-written=%s' % ( - convert_bytes(io.read_bytes), - convert_bytes(io.write_bytes))) - if children: - print_('children', '') - for child in children: - print_('', 'pid=%s name=%s' % (child.pid, child.name())) - - if pinfo['open_files'] != ACCESS_DENIED and pinfo['open_files']: - print_('open files', '') - for file in pinfo['open_files']: - print_('', 'fd=%s %s ' % (file.fd, file.path)) + print_('nice', pinfo['nice']) + if hasattr(proc, "ionice"): + try: + ionice = proc.ionice() + except psutil.Error: + pass + else: + if psutil.WINDOWS: + print_("ionice", ionice) + else: + print_( + "ionice", + f"class={ionice.ioclass}, value={ionice.value}", + ) - if pinfo['threads'] and len(pinfo['threads']) > 1: - print_('running threads', '') - for thread in pinfo['threads']: - print_('', 'id=%s, user-time=%s, sys-time=%s' % ( - thread.id, thread.user_time, thread.system_time)) - if pinfo['connections'] not in (ACCESS_DENIED, []): - print_('open connections', '') - for conn in pinfo['connections']: + print_('num-threads', pinfo['num_threads']) + if psutil.POSIX: + print_('num-fds', pinfo['num_fds']) + if psutil.WINDOWS: + print_('num-handles', pinfo['num_handles']) + + if 'io_counters' in pinfo: + print_('I/O', str_ntuple(pinfo['io_counters'], convert_bytes=True)) + if 'num_ctx_switches' in pinfo: + print_("ctx-switches", str_ntuple(pinfo['num_ctx_switches'])) + if pinfo['children']: + template = "{:<6} {}" + print_("children", template.format("PID", "NAME")) + for child in pinfo['children']: + try: + print_("", template.format(child.pid, child.name())) + except psutil.AccessDenied: + print_("", template.format(child.pid, "")) + except psutil.NoSuchProcess: + pass + + if pinfo['open_files']: + print_('open-files', 'PATH') + for i, file in enumerate(pinfo['open_files']): + if not verbose and i >= NON_VERBOSE_ITERATIONS: + print_("", "[...]") + break + print_('', file.path) + else: + print_('open-files', '') + + if pinfo['net_connections']: + template = "{:<5} {:<25} {:<25} {}" + print_( + 'connections', + template.format("PROTO", "LOCAL ADDR", "REMOTE ADDR", "STATUS"), + ) + for conn in pinfo['net_connections']: if conn.type == socket.SOCK_STREAM: type = 'TCP' elif conn.type == socket.SOCK_DGRAM: @@ -147,19 +258,84 @@ def run(pid): rip, rport = '*', '*' else: rip, rport = conn.raddr - print_('', '%s:%s -> %s:%s type=%s status=%s' % ( - lip, lport, rip, rport, type, conn.status)) - + line = template.format( + type, + f"{lip}:{lport}", + f"{rip}:{rport}", + conn.status, + ) + print_('', line) + else: + print_('connections', '') -def main(argv=None): - if argv is None: - argv = sys.argv - if len(argv) == 1: - sys.exit(run(os.getpid())) - elif len(argv) == 2: - sys.exit(run(int(argv[1]))) + if pinfo['threads'] and len(pinfo['threads']) > 1: + template = "{:<5} {:>12} {:>12}" + print_("threads", template.format("TID", "USER", "SYSTEM")) + for i, thread in enumerate(pinfo['threads']): + if not verbose and i >= NON_VERBOSE_ITERATIONS: + print_("", "[...]") + break + print_("", template.format(*thread)) + print_('', f"total={len(pinfo['threads'])}") else: - sys.exit('usage: %s [pid]' % __file__) + print_('threads', '') + + if hasattr(proc, "rlimit"): + res_names = [x for x in dir(psutil) if x.startswith("RLIMIT")] + resources = [] + for res_name in res_names: + try: + soft, hard = proc.rlimit(getattr(psutil, res_name)) + except psutil.AccessDenied: + pass + else: + resources.append((res_name, soft, hard)) + if resources: + template = "{:<12} {:>15} {:>15}" + print_("res-limits", template.format("RLIMIT", "SOFT", "HARD")) + for res_name, soft, hard in resources: + if soft == psutil.RLIM_INFINITY: + soft = "infinity" + if hard == psutil.RLIM_INFINITY: + hard = "infinity" + print_( + '', + template.format( + RLIMITS_MAP.get(res_name, res_name), soft, hard + ), + ) + + if hasattr(proc, "environ") and pinfo['environ']: + template = "{:<25} {}" + print_("environ", template.format("NAME", "VALUE")) + for i, k in enumerate(sorted(pinfo['environ'])): + if not verbose and i >= NON_VERBOSE_ITERATIONS: + print_("", "[...]") + break + print_("", template.format(k, pinfo["environ"][k])) + + if pinfo.get('memory_maps', None): + template = "{:<8} {}" + print_("mem-maps", template.format("RSS", "PATH")) + maps = sorted(pinfo['memory_maps'], key=lambda x: x.rss, reverse=True) + for i, region in enumerate(maps): + if not verbose and i >= NON_VERBOSE_ITERATIONS: + print_("", "[...]") + break + print_("", template.format(bytes2human(region.rss), region.path)) + + +def main(): + parser = argparse.ArgumentParser( + description="print information about a process" + ) + parser.add_argument("pid", type=int, help="process pid", nargs='?') + parser.add_argument( + '--verbose', '-v', action='store_true', help="print more info" + ) + args = parser.parse_args() + run(args.pid, args.verbose) + if __name__ == '__main__': sys.exit(main()) diff --git a/scripts/procsmem.py b/scripts/procsmem.py index 494fd6aba5..6ccb08b593 100755 --- a/scripts/procsmem.py +++ b/scripts/procsmem.py @@ -1,11 +1,10 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -Show detailed memory usage about all (querable) processes. +"""Show detailed memory usage about all (querable) processes. Processes are sorted by their "USS" (Unique Set Size) memory, which is probably the most representative metric for determining how much memory @@ -33,16 +32,15 @@ 20513 giampao /opt/sublime_text/sublime_text 65.8M 73.0M 0B 87.9M 3976 giampao compiz 115.0M 117.0M 0B 130.9M 32486 giampao skype 145.1M 147.5M 0B 149.6M + """ -from __future__ import print_function import sys import psutil - -if not (psutil.LINUX or psutil.OSX or psutil.WINDOWS): - sys.exit("platform not supported") +if not hasattr(psutil.Process, "memory_footprint"): + sys.exit("can't retrieve USS memory on this platform") def convert_bytes(n): @@ -53,49 +51,56 @@ def convert_bytes(n): for s in reversed(symbols): if n >= prefix[s]: value = float(n) / prefix[s] - return '%.1f%s' % (value, s) - return "%sB" % n + return f"{value:.1f}{s}" + return f"{n}B" def main(): ad_pids = [] procs = [] for p in psutil.process_iter(): - try: - mem = p.memory_full_info() - info = p.as_dict(attrs=["cmdline", "username"]) - except psutil.AccessDenied: - ad_pids.append(p.pid) - except psutil.NoSuchProcess: - pass - else: - p._uss = mem.uss - p._rss = mem.rss - if not p._uss: - continue - p._pss = getattr(mem, "pss", "") - p._swap = getattr(mem, "swap", "") - p._info = info - procs.append(p) + with p.oneshot(): + try: + mem = p.memory_footprint() + info = p.as_dict(["cmdline", "username", "memory_info"]) + except psutil.AccessDenied: + ad_pids.append(p.pid) + except psutil.NoSuchProcess: + pass + else: + p._uss = mem.uss + p._rss = info["memory_info"].rss + p._vms = info["memory_info"].vms + if not p._uss: + continue + p._pss = getattr(mem, "pss", "") + p._swap = getattr(mem, "swap", "") + p._info = info + procs.append(p) procs.sort(key=lambda p: p._uss) - templ = "%-7s %-7s %-30s %7s %7s %7s %7s" - print(templ % ("PID", "User", "Cmdline", "USS", "PSS", "Swap", "RSS")) - print("=" * 78) - for p in procs: - line = templ % ( + templ = "{:<7} {:<7} {:>7} {:>7} {:>7} {:>7} {:>7} {}" + header = templ.format( + "PID", "User", "USS", "PSS", "Swap", "RSS", "VMS", "Cmdline" + ) + print(header) + print("=" * len(header)) + for p in procs[:86]: + cmd = " ".join(p._info["cmdline"])[:50] if p._info["cmdline"] else "" + line = templ.format( p.pid, - p._info["username"][:7], - " ".join(p._info["cmdline"])[:30], + p._info["username"][:7] if p._info["username"] else "", convert_bytes(p._uss), - convert_bytes(p._pss) if p._pss != "" else "", - convert_bytes(p._swap) if p._swap != "" else "", + convert_bytes(p._pss) if p._pss else "", + convert_bytes(p._swap) if p._swap else "", convert_bytes(p._rss), + convert_bytes(p._vms), + cmd, ) print(line) if ad_pids: - print("warning: access denied for %s pids" % (len(ad_pids)), - file=sys.stderr) + print(f"warning: access denied for {len(ad_pids)} pids") + if __name__ == '__main__': sys.exit(main()) diff --git a/scripts/ps.py b/scripts/ps.py index 8aa3d3f495..fe4a7ac770 100755 --- a/scripts/ps.py +++ b/scripts/ps.py @@ -1,102 +1,115 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -A clone of 'ps -aux' on UNIX. +"""A clone of 'ps aux'. -$ python scripts/ps.py -... +$ python3 scripts/ps.py +USER PID %MEM VSZ RSS NICE STATUS START TIME CMDLINE +root 1 0.0 220.9M 6.5M sleep Mar27 09:10 /lib/systemd +root 2 0.0 0.0B 0.0B sleep Mar27 00:00 kthreadd +root 4 0.0 0.0B 0.0B -20 idle Mar27 00:00 kworker/0:0H +root 6 0.0 0.0B 0.0B -20 idle Mar27 00:00 mm_percpu_wq +root 7 0.0 0.0B 0.0B sleep Mar27 00:06 ksoftirqd/0 +root 8 0.0 0.0B 0.0B idle Mar27 03:32 rcu_sched +root 9 0.0 0.0B 0.0B idle Mar27 00:00 rcu_bh +root 10 0.0 0.0B 0.0B sleep Mar27 00:00 migration/0 +root 11 0.0 0.0B 0.0B sleep Mar27 00:00 watchdog/0 +root 12 0.0 0.0B 0.0B sleep Mar27 00:00 cpuhp/0 +root 13 0.0 0.0B 0.0B sleep Mar27 00:00 cpuhp/1 +root 14 0.0 0.0B 0.0B sleep Mar27 00:01 watchdog/1 +root 15 0.0 0.0B 0.0B sleep Mar27 00:00 migration/1 +[...] +giampaolo 19704 1.5 1.9G 235.6M sleep 17:39 01:11 firefox +root 20414 0.0 0.0B 0.0B idle Apr04 00:00 kworker/4:2 +giampaolo 20952 0.0 10.7M 100.0K sleep Mar28 00:00 sh -c /usr +giampaolo 20953 0.0 269.0M 528.0K sleep Mar28 00:00 /usr/lib/ +giampaolo 22150 3.3 2.4G 525.5M sleep Apr02 49:09 /usr/lib/ +root 22338 0.0 0.0B 0.0B idle 02:04 00:00 kworker/1:2 +giampaolo 24123 0.0 35.0M 7.0M sleep 02:12 00:02 bash """ import datetime -import os +import shutil import time import psutil - -PROC_STATUSES_RAW = { - psutil.STATUS_RUNNING: "R", - psutil.STATUS_SLEEPING: "S", - psutil.STATUS_DISK_SLEEP: "D", - psutil.STATUS_STOPPED: "T", - psutil.STATUS_TRACING_STOP: "t", - psutil.STATUS_ZOMBIE: "Z", - psutil.STATUS_DEAD: "X", - psutil.STATUS_WAKING: "WA", - psutil.STATUS_IDLE: "I", - psutil.STATUS_LOCKED: "L", - psutil.STATUS_WAITING: "W", -} - -if hasattr(psutil, 'STATUS_WAKE_KILL'): - PROC_STATUSES_RAW[psutil.STATUS_WAKE_KILL] = "WK" - -if hasattr(psutil, 'STATUS_SUSPENDED'): - PROC_STATUSES_RAW[psutil.STATUS_SUSPENDED] = "V" +from psutil import bytes2human def main(): today_day = datetime.date.today() - templ = "%-10s %5s %4s %4s %7s %7s %-13s %-5s %5s %7s %s" - attrs = ['pid', 'cpu_percent', 'memory_percent', 'name', 'cpu_times', - 'create_time', 'memory_info', 'status'] - if os.name == 'posix': - attrs.append('uids') - attrs.append('terminal') - print(templ % ("USER", "PID", "%CPU", "%MEM", "VSZ", "RSS", "TTY", - "STAT", "START", "TIME", "COMMAND")) - for p in psutil.process_iter(): - try: - pinfo = p.as_dict(attrs, ad_value='') - except psutil.NoSuchProcess: - pass - else: - if pinfo['create_time']: - ctime = datetime.datetime.fromtimestamp(pinfo['create_time']) - if ctime.date() == today_day: - ctime = ctime.strftime("%H:%M") - else: - ctime = ctime.strftime("%b%d") + # fmt: off + templ = "{:<10} {:>5} {:>5} {:>7} {:>7} {:>5} {:>6} {:>6} {:>6} {}" + attrs = ['pid', 'memory_percent', 'name', 'cmdline', 'cpu_times', + 'create_time', 'memory_info', 'status', 'nice', 'username'] + print(templ.format("USER", "PID", "%MEM", "VSZ", "RSS", "NICE", + "STATUS", "START", "TIME", "CMDLINE")) + # fmt: on + for p in psutil.process_iter(attrs, ad_value=None): + if p.create_time(): + ctime = datetime.datetime.fromtimestamp(p.create_time()) + if ctime.date() == today_day: + ctime = ctime.strftime("%H:%M") else: - ctime = '' - cputime = time.strftime("%M:%S", - time.localtime(sum(pinfo['cpu_times']))) + ctime = ctime.strftime("%b%d") + else: + ctime = '' + if p.cpu_times(): + cputime = time.strftime( + "%M:%S", time.localtime(sum(p.cpu_times())) + ) + else: + cputime = '' + + user = p.username() + if not user and psutil.POSIX: try: - user = p.username() - except KeyError: - if os.name == 'posix': - if pinfo['uids']: - user = str(pinfo['uids'].real) - else: - user = '' - else: - raise + user = p.uids()[0] except psutil.Error: - user = '' - if os.name == 'nt' and '\\' in user: - user = user.split('\\')[1] - vms = pinfo['memory_info'] and \ - int(pinfo['memory_info'].vms / 1024) or '?' - rss = pinfo['memory_info'] and \ - int(pinfo['memory_info'].rss / 1024) or '?' - memp = pinfo['memory_percent'] and \ - round(pinfo['memory_percent'], 1) or '?' - status = PROC_STATUSES_RAW.get(pinfo['status'], pinfo['status']) - print(templ % ( - user[:10], - pinfo['pid'], - pinfo['cpu_percent'], - memp, - vms, - rss, - pinfo.get('terminal', '') or '?', - status, - ctime, - cputime, - pinfo['name'].strip() or '?')) + pass + if user and psutil.WINDOWS and '\\' in user: + user = user.split('\\')[1] + if not user: + user = '' + user = user[:9] + vms = ( + bytes2human(p.memory_info().vms) + if p.memory_info() is not None + else '' + ) + rss = ( + bytes2human(p.memory_info().rss) + if p.memory_info() is not None + else '' + ) + memp = ( + round(p.memory_percent(), 1) + if p.memory_percent() is not None + else '' + ) + nice = int(p.nice()) if p.nice() else '' + if p.cmdline(): # noqa: SIM108 + cmdline = ' '.join(p.cmdline()) + else: + cmdline = p.name() + status = p.status()[:5] if p.status() else '' + + line = templ.format( + user, + p.pid, + memp, + vms, + rss, + nice, + status, + ctime, + cputime, + cmdline, + ) + print(line[: shutil.get_terminal_size()[0]]) if __name__ == '__main__': diff --git a/scripts/pstree.py b/scripts/pstree.py index 8e4c9f9572..694c815e72 100755 --- a/scripts/pstree.py +++ b/scripts/pstree.py @@ -1,14 +1,13 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -Similar to 'ps aux --forest' on Linux, prints the process list +"""Similar to 'ps aux --forest' on Linux, prints the process list as a tree structure. -$ python scripts/pstree.py +$ python3 scripts/pstree.py 0 ? |- 1 init | |- 289 cgmanager @@ -28,7 +27,6 @@ ... """ -from __future__ import print_function import collections import sys diff --git a/scripts/sensors.py b/scripts/sensors.py new file mode 100755 index 0000000000..726b53d52c --- /dev/null +++ b/scripts/sensors.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A clone of 'sensors' utility on Linux printing hardware temperatures, +fans speed and battery info. + +$ python3 scripts/sensors.py +asus + Temperatures: + asus 57.0°C (high=None°C, critical=None°C) + Fans: + cpu_fan 3500 RPM +acpitz + Temperatures: + acpitz 57.0°C (high=108.0°C, critical=108.0°C) +coretemp + Temperatures: + Physical id 0 61.0°C (high=87.0°C, critical=105.0°C) + Core 0 61.0°C (high=87.0°C, critical=105.0°C) + Core 1 59.0°C (high=87.0°C, critical=105.0°C) +Battery: + charge: 84.95% + status: charging + plugged in: yes +""" + +import psutil + + +def secs2hours(secs): + mm, ss = divmod(secs, 60) + hh, mm = divmod(mm, 60) + return f"{int(hh)}:{int(mm):02}:{int(ss):02}" + + +def main(): + if hasattr(psutil, "sensors_temperatures"): + temps = psutil.sensors_temperatures() + else: + temps = {} + fans = psutil.sensors_fans() if hasattr(psutil, "sensors_fans") else {} + if hasattr(psutil, "sensors_battery"): + battery = psutil.sensors_battery() + else: + battery = None + + if not any((temps, fans, battery)): + print("can't read any temperature, fans or battery info") + return + + names = set(list(temps.keys()) + list(fans.keys())) + for name in names: + print(name) + # Temperatures. + if name in temps: + print(" Temperatures:") + for entry in temps[name]: + s = " {:<20} {}°C (high={}°C, critical={}°C)".format( + entry.label or name, + entry.current, + entry.high, + entry.critical, + ) + print(s) + # Fans. + if name in fans: + print(" Fans:") + for entry in fans[name]: + print( + " {:<20} {} RPM".format( + entry.label or name, entry.current + ) + ) + + # Battery. + if battery: + print("Battery:") + print(f" charge: {round(battery.percent, 2)}%") + if battery.power_plugged: + print( + " status: {}".format( + "charging" if battery.percent < 100 else "fully charged" + ) + ) + print(" plugged in: yes") + else: + print(f" left: {secs2hours(battery.secsleft)}") + print(" status: discharging") + print(" plugged in: no") + + +if __name__ == '__main__': + main() diff --git a/scripts/temperatures.py b/scripts/temperatures.py new file mode 100755 index 0000000000..f3486b8817 --- /dev/null +++ b/scripts/temperatures.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A clone of 'sensors' utility on Linux printing hardware temperatures. + +$ python3 scripts/sensors.py +asus + asus 47.0 °C (high = None °C, critical = None °C) + +acpitz + acpitz 47.0 °C (high = 103.0 °C, critical = 103.0 °C) + +coretemp + Physical id 0 54.0 °C (high = 100.0 °C, critical = 100.0 °C) + Core 0 47.0 °C (high = 100.0 °C, critical = 100.0 °C) + Core 1 48.0 °C (high = 100.0 °C, critical = 100.0 °C) + Core 2 47.0 °C (high = 100.0 °C, critical = 100.0 °C) + Core 3 54.0 °C (high = 100.0 °C, critical = 100.0 °C) +""" + +import sys + +import psutil + + +def main(): + if not hasattr(psutil, "sensors_temperatures"): + sys.exit("platform not supported") + temps = psutil.sensors_temperatures() + if not temps: + sys.exit("can't read any temperature") + for name, entries in temps.items(): + print(name) + for entry in entries: + line = " {:<20} {} °C (high = {} °C, critical = %{} °C)".format( + entry.label or name, + entry.current, + entry.high, + entry.critical, + ) + print(line) + print() + + +if __name__ == '__main__': + main() diff --git a/scripts/top.py b/scripts/top.py index 1caa8136d8..031b1faaad 100755 --- a/scripts/top.py +++ b/scripts/top.py @@ -1,99 +1,76 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -A clone of top / htop. +"""A clone of top / htop. Author: Giampaolo Rodola' -$ python scripts/top.py - CPU0 [| ] 4.9% - CPU1 [||| ] 7.8% - CPU2 [ ] 2.0% - CPU3 [||||| ] 13.9% - Mem [||||||||||||||||||| ] 49.8% 4920M/9888M - Swap [ ] 0.0% 0M/0M - Processes: 287 (running=1, sleeping=286, zombie=1) - Load average: 0.34 0.54 0.46 Uptime: 3 days, 10:16:37 - -PID USER NI VIRT RES CPU% MEM% TIME+ NAME ------------------------------------------------------------- -989 giampaol 0 66M 12M 7.4 0.1 0:00.61 python -2083 root 0 506M 159M 6.5 1.6 0:29.26 Xorg -4503 giampaol 0 599M 25M 6.5 0.3 3:32.60 gnome-terminal -3868 giampaol 0 358M 8M 2.8 0.1 23:12.60 pulseaudio -3936 giampaol 0 1G 111M 2.8 1.1 33:41.67 compiz -4401 giampaol 0 536M 141M 2.8 1.4 35:42.73 skype -4047 giampaol 0 743M 76M 1.8 0.8 42:03.33 unity-panel-service -13155 giampaol 0 1G 280M 1.8 2.8 41:57.34 chrome -10 root 0 0B 0B 0.9 0.0 4:01.81 rcu_sched -339 giampaol 0 1G 113M 0.9 1.1 8:15.73 chrome +$ python3 scripts/top.py + CPU0 [|||| ] 10.9% + CPU1 [||||| ] 13.1% + CPU2 [||||| ] 12.8% + CPU3 [|||| ] 11.5% + Mem [||||||||||||||||||||||||||||| ] 73.0% 11017M / 15936M + Swap [ ] 1.3% 276M / 20467M + Processes: 347 (sleeping=273, running=1, idle=73) + Load average: 1.10 1.28 1.34 Uptime: 8 days, 21:15:40 + +PID USER NI VIRT RES CPU% MEM% TIME+ NAME +5368 giampaol 0 7.2G 4.3G 41.8 27.7 56:34.18 VirtualBox +24976 giampaol 0 2.1G 487.2M 18.7 3.1 22:05.16 Web Content +22731 giampaol 0 3.2G 596.2M 11.6 3.7 35:04.90 firefox +1202 root 0 807.4M 288.5M 10.6 1.8 12:22.12 Xorg +22811 giampaol 0 2.8G 741.8M 9.0 4.7 2:26.61 Web Content +2590 giampaol 0 2.3G 579.4M 5.5 3.6 28:02.70 compiz +22990 giampaol 0 3.0G 1.2G 4.2 7.6 4:30.32 Web Content +18412 giampaol 0 90.1M 14.5M 3.5 0.1 0:00.26 python3 +26971 netdata 0 20.8M 3.9M 2.9 0.0 3:17.14 apps.plugin +2421 giampaol 0 3.3G 36.9M 2.3 0.2 57:14.21 pulseaudio ... """ -import atexit import datetime -import os import sys import time + try: import curses except ImportError: sys.exit('platform not supported') import psutil - - -# --- curses stuff -def tear_down(): - win.keypad(0) - curses.nocbreak() - curses.echo() - curses.endwin() +from psutil import bytes2human win = curses.initscr() -atexit.register(tear_down) -curses.endwin() lineno = 0 +colors_map = dict(green=3, red=10, yellow=4) -def print_line(line, highlight=False): +def printl(line, color=None, bold=False, highlight=False): """A thin wrapper around curses's addstr().""" global lineno try: + flags = 0 + if color: + flags |= curses.color_pair(colors_map[color]) + if bold: + flags |= curses.A_BOLD if highlight: line += " " * (win.getmaxyx()[1] - len(line)) - win.addstr(lineno, 0, line, curses.A_REVERSE) - else: - win.addstr(lineno, 0, line, 0) + flags |= curses.A_STANDOUT + win.addstr(lineno, 0, line, flags) except curses.error: lineno = 0 win.refresh() raise else: lineno += 1 -# --- /curses stuff -def bytes2human(n): - """ - >>> bytes2human(10000) - '9K' - >>> bytes2human(100001221) - '95M' - """ - symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') - prefix = {} - for i, s in enumerate(symbols): - prefix[s] = 1 << (i + 1) * 10 - for s in reversed(symbols): - if n >= prefix[s]: - value = int(float(n) / prefix[s]) - return '%s%s' % (value, s) - return "%sB" % n +# --- /curses stuff def poll(interval): @@ -103,9 +80,16 @@ def poll(interval): procs_status = {} for p in psutil.process_iter(): try: - p.dict = p.as_dict(['username', 'nice', 'memory_info', - 'memory_percent', 'cpu_percent', - 'cpu_times', 'name', 'status']) + p.dict = p.as_dict([ + 'username', + 'nice', + 'memory_info', + 'memory_percent', + 'cpu_percent', + 'cpu_times', + 'name', + 'status', + ]) try: procs_status[p.dict['status']] += 1 except KeyError: @@ -116,16 +100,26 @@ def poll(interval): procs.append(p) # return processes sorted by CPU percent usage - processes = sorted(procs, key=lambda p: p.dict['cpu_percent'], - reverse=True) + processes = sorted( + procs, key=lambda p: p.dict['cpu_percent'], reverse=True + ) return (processes, procs_status) +def get_color(perc): + if perc <= 30: + return "green" + elif perc <= 80: + return "yellow" + else: + return "red" + + def print_header(procs_status, num_procs): """Print system-related info, above the process list.""" def get_dashes(perc): - dashes = "|" * int((float(perc) / 10 * 4)) + dashes = "|" * int(float(perc) / 10 * 4) empty_dashes = " " * (40 - len(dashes)) return dashes, empty_dashes @@ -133,64 +127,85 @@ def get_dashes(perc): percs = psutil.cpu_percent(interval=0, percpu=True) for cpu_num, perc in enumerate(percs): dashes, empty_dashes = get_dashes(perc) - print_line(" CPU%-2s [%s%s] %5s%%" % (cpu_num, dashes, empty_dashes, - perc)) + line = " CPU{:<2} [{}{}] {:>5}%".format( + cpu_num, dashes, empty_dashes, perc + ) + printl(line, color=get_color(perc)) + + # memory usage mem = psutil.virtual_memory() dashes, empty_dashes = get_dashes(mem.percent) - used = mem.total - mem.available - line = " Mem [%s%s] %5s%% %6s/%s" % ( - dashes, empty_dashes, + line = " Mem [{}{}] {:>5}% {:>6} / {}".format( + dashes, + empty_dashes, mem.percent, - str(int(used / 1024 / 1024)) + "M", - str(int(mem.total / 1024 / 1024)) + "M" + bytes2human(mem.used), + bytes2human(mem.total), ) - print_line(line) + printl(line, color=get_color(mem.percent)) # swap usage swap = psutil.swap_memory() dashes, empty_dashes = get_dashes(swap.percent) - line = " Swap [%s%s] %5s%% %6s/%s" % ( - dashes, empty_dashes, + line = " Swap [{}{}] {:>5}% {:>6} / {}".format( + dashes, + empty_dashes, swap.percent, - str(int(swap.used / 1024 / 1024)) + "M", - str(int(swap.total / 1024 / 1024)) + "M" + bytes2human(swap.used), + bytes2human(swap.total), ) - print_line(line) + printl(line, color=get_color(swap.percent)) # processes number and status st = [] for x, y in procs_status.items(): if y: - st.append("%s=%s" % (x, y)) - st.sort(key=lambda x: x[:3] in ('run', 'sle'), reverse=1) - print_line(" Processes: %s (%s)" % (num_procs, ', '.join(st))) + st.append(f"{x}={y}") + st.sort(key=lambda x: x[:3] in {'run', 'sle'}, reverse=1) + printl(f" Processes: {num_procs} ({', '.join(st)})") # load average, uptime - uptime = datetime.datetime.now() - \ - datetime.datetime.fromtimestamp(psutil.boot_time()) - av1, av2, av3 = os.getloadavg() - line = " Load average: %.2f %.2f %.2f Uptime: %s" \ - % (av1, av2, av3, str(uptime).split('.')[0]) - print_line(line) + uptime = datetime.datetime.now() - datetime.datetime.fromtimestamp( + psutil.boot_time() + ) + av1, av2, av3 = psutil.getloadavg() + line = " Load average: {:.2f} {:.2f} {:.2f} Uptime: {}".format( + av1, + av2, + av3, + str(uptime).split('.')[0], + ) + printl(line) def refresh_window(procs, procs_status): """Print results on screen by using curses.""" curses.endwin() - templ = "%-6s %-8s %4s %5s %5s %6s %4s %9s %2s" + templ = "{:<6} {:<8} {:>4} {:>6} {:>6} {:>5} {:>5} {:>9} {:>2}" win.erase() - header = templ % ("PID", "USER", "NI", "VIRT", "RES", "CPU%", "MEM%", - "TIME+", "NAME") + header = templ.format( + "PID", + "USER", + "NI", + "VIRT", + "RES", + "CPU%", + "MEM%", + "TIME+", + "NAME", + ) print_header(procs_status, len(procs)) - print_line("") - print_line(header, highlight=True) + printl("") + printl(header, bold=True, highlight=True) for p in procs: # TIME+ column shows process CPU cumulative time and it # is expressed as: "mm:ss.ms" if p.dict['cpu_times'] is not None: ctime = datetime.timedelta(seconds=sum(p.dict['cpu_times'])) - ctime = "%s:%s.%s" % (ctime.seconds // 60 % 60, - str((ctime.seconds % 60)).zfill(2), - str(ctime.microseconds)[:2]) + ctime = "{}:{}.{}".format( + ctime.seconds // 60 % 60, + str(ctime.seconds % 60).zfill(2), + str(ctime.microseconds)[:2], + ) else: ctime = '' if p.dict['memory_percent'] is not None: @@ -199,36 +214,56 @@ def refresh_window(procs, procs_status): p.dict['memory_percent'] = '' if p.dict['cpu_percent'] is None: p.dict['cpu_percent'] = '' - if p.dict['username']: - username = p.dict['username'][:8] - else: - username = "" - line = templ % (p.pid, - username, - p.dict['nice'], - bytes2human(getattr(p.dict['memory_info'], 'vms', 0)), - bytes2human(getattr(p.dict['memory_info'], 'rss', 0)), - p.dict['cpu_percent'], - p.dict['memory_percent'], - ctime, - p.dict['name'] or '', - ) + username = p.dict['username'][:8] if p.dict['username'] else '' + line = templ.format( + p.pid, + username, + p.dict['nice'], + bytes2human(getattr(p.dict['memory_info'], 'vms', 0)), + bytes2human(getattr(p.dict['memory_info'], 'rss', 0)), + p.dict['cpu_percent'], + p.dict['memory_percent'], + ctime, + p.dict['name'] or '', + ) try: - print_line(line) + printl(line) except curses.error: break win.refresh() +def setup(): + curses.start_color() + curses.use_default_colors() + for i in range(curses.COLORS): + curses.init_pair(i + 1, i, -1) + curses.endwin() + win.nodelay(1) + + +def tear_down(): + win.keypad(0) + curses.nocbreak() + curses.echo() + curses.endwin() + + def main(): + setup() try: interval = 0 while True: + if win.getch() == ord('q'): + break args = poll(interval) refresh_window(*args) interval = 1 except (KeyboardInterrupt, SystemExit): pass + finally: + tear_down() + if __name__ == '__main__': main() diff --git a/scripts/who.py b/scripts/who.py index f64c00931a..d459547234 100755 --- a/scripts/who.py +++ b/scripts/who.py @@ -1,18 +1,15 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -A clone of 'who' command; print information about users who are +"""A clone of 'who' command; print information about users who are currently logged in. -$ python scripts/who.py -giampaolo tty7 2014-02-23 17:25 (:0) -giampaolo pts/7 2014-02-24 18:25 (:192.168.1.56) -giampaolo pts/8 2014-02-24 18:25 (:0) -giampaolo pts/9 2014-02-27 01:32 (:0) +$ python3 scripts/who.py +giampaolo console 2017-03-25 22:24 loginwindow +giampaolo ttys000 2017-03-25 23:28 (10.0.2.2) sshd """ from datetime import datetime @@ -23,11 +20,16 @@ def main(): users = psutil.users() for user in users: - print("%-15s %-15s %s (%s)" % ( + proc_name = psutil.Process(user.pid).name() if user.pid else "" + line = "{:<12} {:<10} {:<10} {:<14} {}".format( user.name, user.terminal or '-', datetime.fromtimestamp(user.started).strftime("%Y-%m-%d %H:%M"), - user.host)) + f"({user.host or ''})", + proc_name, + ) + print(line) + if __name__ == '__main__': main() diff --git a/scripts/winservices.py b/scripts/winservices.py index fed6a734ef..bb58287506 100755 --- a/scripts/winservices.py +++ b/scripts/winservices.py @@ -1,13 +1,12 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -List all Windows services installed. +r"""List all Windows services installed. -$ python scripts/winservices.py +$ python3 scripts/winservices.py AeLookupSvc (Application Experience) status: stopped, start: manual, username: localSystem, pid: None binpath: C:\Windows\system32\svchost.exe -k netsvcs @@ -27,29 +26,36 @@ Appinfo (Application Information) status: stopped, start: manual, username: LocalSystem, pid: None binpath: C:\Windows\system32\svchost.exe -k netsvcs - ... """ - import os import sys import psutil - if os.name != 'nt': sys.exit("platform not supported (Windows only)") def main(): for service in psutil.win_service_iter(): + if service.name() == "WaaSMedicSvc": + # known issue in Windows 11 reading the description + # https://learn.microsoft.com/en-us/answers/questions/1320388/in-windows-11-version-22h2-there-it-shows-(failed + # https://github.com/giampaolo/psutil/issues/2383 + continue info = service.as_dict() - print("%s (%s)" % (info['name'], info['display_name'])) - print("status: %s, start: %s, username: %s, pid: %s" % ( - info['status'], info['start_type'], info['username'], info['pid'])) - print("binpath: %s" % info['binpath']) - print("") + print(f"{info['name']!r} ({info['display_name']!r})") + s = "status: {}, start: {}, username: {}, pid: {}".format( + info['status'], + info['start_type'], + info['username'], + info['pid'], + ) + print(s) + print(f"binpath: {info['binpath']}") + print() if __name__ == '__main__': diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index a61b5ed2a0..c5426acb1d --- a/setup.py +++ b/setup.py @@ -1,285 +1,488 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""psutil is a cross-platform library for retrieving information on -running processes and system utilization (CPU, memory, disks, network) -in Python. -""" +"""Cross-platform lib for process and system monitoring in Python.""" -import atexit -import contextlib -import io +import concurrent.futures +import glob import os +import pathlib +import shlex +import shutil +import struct +import subprocess import sys +import sysconfig import tempfile -import platform -try: - from setuptools import setup, Extension -except ImportError: - from distutils.core import setup, Extension - -HERE = os.path.abspath(os.path.dirname(__file__)) -sys.path.insert(0, os.path.join(HERE, "psutil")) -import _common # NOQA - - -def get_version(): - INIT = os.path.join(HERE, 'psutil/__init__.py') - with open(INIT, 'r') as f: - for line in f: - if line.startswith('__version__'): - ret = eval(line.strip().split(' = ')[1]) - assert ret.count('.') == 2, ret - for num in ret.split('.'): - assert num.isdigit(), ret - return ret - else: - raise ValueError("couldn't find version string") +from setuptools import Extension +from setuptools import setup +from setuptools.command.build_ext import build_ext -def get_description(): - README = os.path.join(HERE, 'README.rst') - with open(README, 'r') as f: - return f.read() +ROOT_DIR = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT_DIR)) +from _bootstrap import get_version # noqa: E402 +from _bootstrap import load_module # noqa: E402 -@contextlib.contextmanager -def silenced_output(stream_name): - class DummyFile(io.BytesIO): - # see: https://github.com/giampaolo/psutil/issues/678 - errors = "ignore" +_common = load_module(ROOT_DIR / "psutil" / "_common.py") - def write(self, s): - pass +AIX = _common.AIX +BSD = _common.BSD +FREEBSD = _common.FREEBSD +LINUX = _common.LINUX +MACOS = _common.MACOS +NETBSD = _common.NETBSD +OPENBSD = _common.OPENBSD +POSIX = _common.POSIX +SUNOS = _common.SUNOS +WINDOWS = _common.WINDOWS - orig = getattr(sys, stream_name) - try: - setattr(sys, stream_name, DummyFile()) - yield - finally: - setattr(sys, stream_name, orig) +hilite = _common.hilite + +PYPY = '__pypy__' in sys.builtin_module_names +CPYTHON = sys.implementation.name == "cpython" +Py_GIL_DISABLED = sysconfig.get_config_var("Py_GIL_DISABLED") + + +# The pre-processor macros that are passed to the C compiler when +# building the extension. +macros = [] + +if POSIX: + macros.append(("PSUTIL_POSIX", 1)) +if BSD: + macros.append(("PSUTIL_BSD", 1)) + +# Needed to determine _Py_PARSE_PID in case it's missing (PyPy). +# Taken from Lib/test/test_fcntl.py. +# XXX: not bullet proof as the (long long) case is missing. +if struct.calcsize('l') <= 8: + macros.append(('PSUTIL_SIZEOF_PID_T', '4')) # int +else: + macros.append(('PSUTIL_SIZEOF_PID_T', '8')) # long + + +sources = glob.glob("psutil/arch/all/*.c") +if POSIX: + sources.extend(glob.glob("psutil/arch/posix/*.c")) VERSION = get_version() -VERSION_MACRO = ('PSUTIL_VERSION', int(VERSION.replace('.', ''))) - - -# POSIX -if _common.POSIX: - posix_extension = Extension( - 'psutil._psutil_posix', - sources=['psutil/_psutil_posix.c']) - if sys.platform.startswith("sunos") or sys.platform.startswith("solaris"): - posix_extension.libraries.append('socket') - if platform.release() == '5.10': - posix_extension.sources.append('psutil/arch/solaris/v10/ifaddrs.c') - posix_extension.define_macros.append(('PSUTIL_SUNOS10', 1)) -# Windows -if _common.WINDOWS: +macros.append(('PSUTIL_VERSION', int(VERSION.replace('.', '')))) + +# The oldest interpreter we support, and the one the wheel claims to +# run on. Py_LIMITED_API lets us create a single wheel which works with +# multiple python versions, including unreleased ones. +MIN_PY_VERSION = (3, 8) + +abi3_platform = MACOS or LINUX or WINDOWS # the ones we ship wheels for +if CPYTHON and abi3_platform and not Py_GIL_DISABLED: + _abi3_tag = "cp{}{}".format(*MIN_PY_VERSION) + _hexversion = "0x{:02x}{:02x}0000".format(*MIN_PY_VERSION) + py_limited_api = {"py_limited_api": True} + options = {"bdist_wheel": {"py_limited_api": _abi3_tag}} + macros.append(('Py_LIMITED_API', _hexversion)) +else: + py_limited_api = {} + options = {} + + +def get_long_description(): + script = ROOT_DIR / "scripts" / "internal" / "convert_readme.py" + readme = ROOT_DIR / 'README.rst' + p = subprocess.Popen( + [sys.executable, script, readme], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = p.communicate() + if p.returncode != 0: + raise RuntimeError(stderr) + return stdout + + +def num_cpus(): + value = os.getenv("PSUTIL_BUILD_JOBS") + if value is not None: + return max(1, int(value)) + fun = getattr(os, "process_cpu_count", os.cpu_count) + return fun() or 1 + + +def has_python_h(): + """Whether a C file including Python.h really compiles.""" + paths = sysconfig.get_paths() + incdirs = [paths["include"]] + if paths.get("platinclude") and paths["platinclude"] not in incdirs: + incdirs.append(paths["platinclude"]) + args = [] + for d in incdirs: + args.extend(["-I", d]) + return unix_can_compile("#include ", args) + + +def get_cc(): + """The compiler (plus flags) python uses to build C extensions.""" + cc = os.getenv('CC') or sysconfig.get_config_var("CC") or "cc" + return shlex.split(cc) + + +def has_compiler(): + return unix_can_compile("int main(void) { return 0; }") + + +def unix_can_compile(c_code, extra_args=()): + # https://github.com/giampaolo/psutil/pull/1568 + with tempfile.TemporaryDirectory() as tempdir: + src = os.path.join(tempdir, "test.c") + with open(src, "w") as f: + f.write(c_code) + cmd = ( + get_cc() + + list(extra_args) + + [ + "-c", + src, + "-o", + os.path.join(tempdir, "test.o"), + ] + ) + try: + ret = subprocess.call( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + except OSError: + return False # compiler is not installed + return ret == 0 + + +if WINDOWS: + def get_winver(): maj, min = sys.getwindowsversion()[0:2] - return '0x0%s' % ((maj * 100) + min) + return "0x0{}".format((maj * 100) + min) + + if sys.getwindowsversion()[0] < 10: + msg = "this Windows version is too old (< Windows 10); " + msg += "psutil 7.2.x is the latest version which supports Windows " + msg += "Vista, 7, 8, 8.1 and their server counterparts" + raise RuntimeError(msg) + + macros.append(("PSUTIL_WINDOWS", 1)) + macros.extend([ + # be nice to mingw, see: + # http://www.mingw.org/wiki/Use_more_recent_defined_functions + ('_WIN32_WINNT', get_winver()), + ('_AVAIL_WINVER_', get_winver()), + ('_CRT_SECURE_NO_WARNINGS', None), + ]) + + if Py_GIL_DISABLED: + macros.append(('Py_GIL_DISABLED', 1)) ext = Extension( - 'psutil._psutil_windows', - sources=[ - 'psutil/_psutil_windows.c', - 'psutil/_psutil_common.c', - 'psutil/arch/windows/process_info.c', - 'psutil/arch/windows/process_handles.c', - 'psutil/arch/windows/security.c', - 'psutil/arch/windows/inet_ntop.c', - 'psutil/arch/windows/services.c', - ], - define_macros=[ - VERSION_MACRO, - # be nice to mingw, see: - # http://www.mingw.org/wiki/Use_more_recent_defined_functions - ('_WIN32_WINNT', get_winver()), - ('_AVAIL_WINVER_', get_winver()), - ('_CRT_SECURE_NO_WARNINGS', None), - # see: https://github.com/giampaolo/psutil/issues/348 - ('PSAPI_VERSION', 1), - ], + 'psutil._psutil', + sources=( + sources + + ["psutil/_psutil_windows.c"] + + glob.glob("psutil/arch/windows/*.c") + ), + define_macros=macros, libraries=[ - "psapi", "kernel32", "advapi32", "shell32", "netapi32", - "iphlpapi", "wtsapi32", "ws2_32", + "advapi32", + "iphlpapi", + "kernel32", + "netapi32", + "ntdll", + "pdh", + "PowrProf", + "shell32", + "ws2_32", ], - # extra_compile_args=["/Z7"], - # extra_link_args=["/DEBUG"] + # extra_compile_args=["/W 4"], + # extra_link_args=["/DEBUG"], + **py_limited_api, ) - extensions = [ext] -# OS X -elif _common.OSX: + +elif MACOS: + macros.extend([("PSUTIL_OSX", 1), ("PSUTIL_MACOS", 1)]) ext = Extension( - 'psutil._psutil_osx', - sources=[ - 'psutil/_psutil_osx.c', - 'psutil/_psutil_common.c', - 'psutil/arch/osx/process_info.c', - ], - define_macros=[VERSION_MACRO], + 'psutil._psutil', + sources=( + sources + + ["psutil/_psutil_osx.c"] + + glob.glob("psutil/arch/osx/*.c") + ), + define_macros=macros, extra_link_args=[ - '-framework', 'CoreFoundation', '-framework', 'IOKit' - ]) - extensions = [ext, posix_extension] -# FreeBSD -elif _common.FREEBSD: - ext = Extension( - 'psutil._psutil_bsd', - sources=[ - 'psutil/_psutil_bsd.c', - 'psutil/_psutil_common.c', - 'psutil/arch/bsd/freebsd.c', - 'psutil/arch/bsd/freebsd_socks.c', + '-framework', + 'CoreFoundation', + '-framework', + 'IOKit', ], - define_macros=[VERSION_MACRO], - libraries=["devstat"]) - extensions = [ext, posix_extension] -# OpenBSD -elif _common.OPENBSD: + **py_limited_api, + ) + +elif FREEBSD: + macros.append(("PSUTIL_FREEBSD", 1)) + ext = Extension( - 'psutil._psutil_bsd', - sources=[ - 'psutil/_psutil_bsd.c', - 'psutil/_psutil_common.c', - 'psutil/arch/bsd/openbsd.c', - ], - define_macros=[VERSION_MACRO], - libraries=["kvm"]) - extensions = [ext, posix_extension] -# NetBSD -elif _common.NETBSD: + 'psutil._psutil', + sources=( + sources + + ["psutil/_psutil_bsd.c"] + + glob.glob("psutil/arch/bsd/*.c") + + glob.glob("psutil/arch/freebsd/*.c") + ), + define_macros=macros, + libraries=["devstat"], + **py_limited_api, + ) + +elif OPENBSD: + macros.append(("PSUTIL_OPENBSD", 1)) + ext = Extension( - 'psutil._psutil_bsd', - sources=[ - 'psutil/_psutil_bsd.c', - 'psutil/_psutil_common.c', - 'psutil/arch/bsd/netbsd.c', - 'psutil/arch/bsd/netbsd_socks.c', - ], - define_macros=[VERSION_MACRO], - libraries=["kvm"]) - extensions = [ext, posix_extension] -# Linux -elif _common.LINUX: - def get_ethtool_macro(): - # see: https://github.com/giampaolo/psutil/issues/659 - from distutils.unixccompiler import UnixCCompiler - from distutils.errors import CompileError - - with tempfile.NamedTemporaryFile( - suffix='.c', delete=False, mode="wt") as f: - f.write("#include ") - - @atexit.register - def on_exit(): - try: - os.remove(f.name) - except OSError: - pass - - compiler = UnixCCompiler() - try: - with silenced_output('stderr'): - with silenced_output('stdout'): - compiler.compile([f.name]) - except CompileError: - return ("PSUTIL_ETHTOOL_MISSING_TYPES", 1) - else: - return None + 'psutil._psutil', + sources=( + sources + + ["psutil/_psutil_bsd.c"] + + glob.glob("psutil/arch/bsd/*.c") + + glob.glob("psutil/arch/openbsd/*.c") + ), + define_macros=macros, + libraries=["kvm"], + **py_limited_api, + ) + +elif NETBSD: + macros.append(("PSUTIL_NETBSD", 1)) + + ext = Extension( + 'psutil._psutil', + sources=( + sources + + ["psutil/_psutil_bsd.c"] + + glob.glob("psutil/arch/bsd/*.c") + + glob.glob("psutil/arch/netbsd/*.c") + ), + define_macros=macros, + libraries=["kvm", "jemalloc"], + **py_limited_api, + ) + +elif LINUX: + # see: https://github.com/giampaolo/psutil/issues/659 + if not unix_can_compile("#include "): + macros.append(("PSUTIL_ETHTOOL_MISSING_TYPES", 1)) - ETHTOOL_MACRO = get_ethtool_macro() - macros = [VERSION_MACRO] - if ETHTOOL_MACRO is not None: - macros.append(ETHTOOL_MACRO) + macros.append(("PSUTIL_LINUX", 1)) ext = Extension( - 'psutil._psutil_linux', - sources=['psutil/_psutil_linux.c'], - define_macros=macros) - extensions = [ext, posix_extension] -# Solaris -elif _common.SUNOS: + 'psutil._psutil', + sources=( + sources + + ["psutil/_psutil_linux.c"] + + glob.glob("psutil/arch/linux/*.c") + ), + define_macros=macros, + **py_limited_api, + ) + +elif SUNOS: + macros.append(("PSUTIL_SUNOS", 1)) + ext = Extension( - 'psutil._psutil_sunos', - sources=['psutil/_psutil_sunos.c'], - define_macros=[VERSION_MACRO], - libraries=['kstat', 'nsl', 'socket']) - extensions = [ext, posix_extension] + 'psutil._psutil', + sources=( + sources + + ["psutil/_psutil_sunos.c"] + + glob.glob("psutil/arch/sunos/*.c") + ), + define_macros=macros, + libraries=["kstat", "nsl", "socket"], + **py_limited_api, + ) + +elif AIX: + macros.append(("PSUTIL_AIX", 1)) + + ext = Extension( + 'psutil._psutil', + sources=( + sources + + ["psutil/_psutil_aix.c"] + + glob.glob("psutil/arch/aix/*.c") + ), + libraries=["perfstat"], + define_macros=macros, + **py_limited_api, + ) + else: - sys.exit('platform %s is not supported' % sys.platform) + sys.exit("platform {} is not supported".format(sys.platform)) + + +class BuildExt(build_ext): + """Compile the C sources in parallel.""" + + def build_extensions(self): # override + compiler = self.compiler + real_spawn = compiler.spawn + real_compile = compiler.compile + + def parallel_compile(*args, **kwargs): + # Run compile() as usual, but have every compiler + # invocation return right away, then wait for all of them. + # Hooking spawn() instead of the private per-file methods + # is what makes this work on Windows as well. + futures = [] + with concurrent.futures.ThreadPoolExecutor(num_cpus()) as pool: + compiler.spawn = lambda cmd, **kw: futures.append( + pool.submit(real_spawn, cmd, **kw) + ) + try: + objects = real_compile(*args, **kwargs) + finally: + compiler.spawn = real_spawn + for fut in concurrent.futures.as_completed(futures): + fut.result() # let compiler errors surface + return objects + + compiler.compile = parallel_compile + super().build_extensions() + + +def print_install_instructions(): + + def install_sysdeps_cmd(): + url = ( + "https://raw.githubusercontent.com/giampaolo/psutil/" + "master/scripts/internal/install-sysdeps.sh" + ) + if shutil.which("curl"): + return f"curl -fsSL {url} | sh" + if shutil.which("wget"): + return f"wget -qO- {url} | sh" + if shutil.which("fetch"): # FreeBSD + return f"fetch -qo - {url} | sh" + if shutil.which("ftp"): # OpenBSD / NetBSD + return f"ftp -o - {url} | sh" + + if not has_compiler(): + suggest = "A working C compiler is not installed." + if MACOS: + cmd = "xcode-select --install" + elif AIX or PYPY: + cmd = None + else: + cmd = install_sysdeps_cmd() + elif not has_python_h(): + suggest = "Python header files are not installed." + if MACOS or AIX or PYPY: # noqa: SIM108 + cmd = None + else: + cmd = install_sysdeps_cmd() + else: + return + + if cmd: + suggest += f" Try running:\n{cmd}" + + print(hilite(suggest, color="red", bold=True), file=sys.stderr) def main(): - setup_args = dict( + kwargs = dict( name='psutil', version=VERSION, - description=__doc__.replace('\n', '').strip(), - long_description=get_description(), + description="Cross-platform lib for process and system monitoring.", + long_description=get_long_description(), + long_description_content_type='text/x-rst', + # fmt: off keywords=[ - 'ps', 'top', 'kill', 'free', 'lsof', 'netstat', 'nice', 'tty', - 'ionice', 'uptime', 'taskmgr', 'process', 'df', 'iotop', 'iostat', - 'ifconfig', 'taskset', 'who', 'pidof', 'pmap', 'smem', 'pstree', - 'monitoring', 'ulimit', 'prlimit', 'smem', + 'ps', 'top', 'kill', 'free', 'lsof', 'netstat', 'df', 'uptime', + 'taskmgr', 'process', 'monitoring', 'performance', 'metrics', + 'observability', ], + # fmt: on author='Giampaolo Rodola', - author_email='g.rodola gmail com', + author_email='g.rodola@gmail.com', url='https://github.com/giampaolo/psutil', platforms='Platform Independent', - license='BSD', - packages=['psutil', 'psutil.tests'], - # see: python setup.py register --list-classifiers + license='BSD-3-Clause', + packages=['psutil'], + ext_modules=[ext], + cmdclass={'build_ext': BuildExt if num_cpus() > 1 else build_ext}, + options=options, + python_requires=">={}.{}".format(*MIN_PY_VERSION), + # https://docs.pypi.org/project_metadata/ + project_urls={ + 'Homepage': 'https://github.com/giampaolo/psutil', + 'Source': 'https://github.com/giampaolo/psutil', + 'Issues': 'https://github.com/giampaolo/psutil/issues', + 'Documentation': 'https://psutil.io/', + 'Changelog': 'https://psutil.io/changelog/', + 'Funding': 'https://github.com/sponsors/giampaolo', + }, + # https://pypi.org/classifiers/ classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', - 'Environment :: Win32 (MS Windows)', 'Intended Audience :: Developers', 'Intended Audience :: Information Technology', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: BSD License', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft :: Windows :: Windows NT/2000', - 'Operating System :: Microsoft', 'Operating System :: OS Independent', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: Microsoft :: Windows :: Windows 10', + 'Operating System :: Microsoft :: Windows :: Windows 11', + 'Operating System :: POSIX :: AIX', + 'Operating System :: POSIX :: BSD', 'Operating System :: POSIX :: BSD :: FreeBSD', 'Operating System :: POSIX :: BSD :: NetBSD', 'Operating System :: POSIX :: BSD :: OpenBSD', - 'Operating System :: POSIX :: BSD', 'Operating System :: POSIX :: Linux', 'Operating System :: POSIX :: SunOS/Solaris', 'Operating System :: POSIX', 'Programming Language :: C', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.0', - 'Programming Language :: Python :: 3.1', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', - 'Programming Language :: Python', - 'Topic :: Software Development :: Libraries :: Python Modules', + 'Programming Language :: Python :: Free Threading', 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: System :: Benchmark', 'Topic :: System :: Hardware', 'Topic :: System :: Monitoring', 'Topic :: System :: Networking :: Monitoring', + 'Topic :: System :: Networking :: Monitoring :: Hardware Watchdog', 'Topic :: System :: Networking', 'Topic :: System :: Operating System', 'Topic :: System :: Systems Administration', 'Topic :: Utilities', ], ) - if extensions is not None: - setup_args["ext_modules"] = extensions - setup(**setup_args) + success = False + try: + setup(**kwargs) + success = True + finally: + cmd = sys.argv[1] if len(sys.argv) >= 2 else '' + if ( + not success + and POSIX + and cmd.startswith(("build", "install", "bdist", "develop")) + ): + print_install_instructions() + if __name__ == '__main__': main() diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000..a1445853a4 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,38 @@ +# Instructions for running tests + +Install deps (e.g. pytest): + +```bash +make install-pydeps-test +``` + +Some tests shell out to CLI tools (`ps`, `ifconfig`, ...). On UNIX install them +with: + +```bash +make install-sysdeps-test +``` + +Run tests: + +```bash +make test +``` + +Run tests in parallel (faster): + +```bash +make test-parallel +``` + +Run a specific test: + +```bash +make test ARGS=tests/test_system.py::TestDiskAPIs +``` + +Test C extension memory leaks: + +```bash +make test-memleaks +``` diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e60f8d4c6d --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,2063 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Test utilities.""" + +import atexit +import contextlib +import ctypes +import enum +import errno +import functools +import importlib.util +import ipaddress +import os +import pathlib +import platform +import random +import re +import select +import shlex +import shutil +import signal +import socket +import stat +import subprocess +import sys +import sysconfig +import tempfile +import textwrap +import threading +import time +import traceback +import types +import typing +import unittest +import warnings +from socket import AF_INET +from socket import AF_INET6 +from socket import SOCK_STREAM + +import pytest + +import psutil +import psutil._ntuples as ntuples +from psutil import AIX +from psutil import BSD +from psutil import LINUX +from psutil import MACOS +from psutil import NETBSD +from psutil import OPENBSD +from psutil import POSIX +from psutil import SUNOS +from psutil import WINDOWS +from psutil import _enums +from psutil._common import ENCODING +from psutil._common import ENCODING_ERRS +from psutil._common import debug +from psutil._common import supports_ipv6 + +if POSIX: + from psutil._psposix import wait_pid + + +# fmt: off +__all__ = [ + # constants + 'DEVNULL', 'GLOBAL_TIMEOUT', 'TOLERANCE_SYS_MEM', 'NO_RETRIES', + 'PYPY', 'PYTHON_EXE', 'PYTHON_EXE_ENV', 'ROOT_DIR', + 'TESTFN_PREFIX', 'UNICODE_SUFFIX', 'INVALID_UNICODE_SUFFIX', + 'CI_TESTING', 'VALID_PROC_STATUSES', 'TOLERANCE_DISK_USAGE', + "HAS_PROC_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_PROC_ENVIRON", + "HAS_PROC_IO_COUNTERS", "HAS_PROC_IONICE", + "HAS_PROC_MEMORY_EXTRAS", "HAS_PROC_MEMORY_FOOTPRINT", + "HAS_PROC_MEMORY_MAPS", + "HAS_PROC_CPU_NUM", "HAS_PROC_RLIMIT", "HAS_SENSORS_BATTERY", + "HAS_BATTERY", "HAS_SENSORS_FANS", "HAS_SENSORS_TEMPERATURES", + "HAS_NET_CONNECTIONS_UNIX", "HAS_PROC_OPEN_FILES_PATH", + "MACOS_11PLUS", "MACOS_12PLUS", "COVERAGE", + "AARCH64", "PYTEST_PARALLEL", + # subprocesses + 'pyrun', 'terminate', 'reap_children', 'spawn_subproc', 'spawn_zombie', + 'spawn_children_pair', 'filter_alien_children', + # threads + 'ThreadTask', + # test utils + 'unittest', 'skip_on_access_denied', 'skip_on_not_implemented', + 'retry_on_failure', 'PsutilTestCase', 'process_namespace', + 'system_namespace', 'is_win_secure_system_proc', 'serial', 'isolated', + 'skipif', 'requires_cli', + # type hints + 'check_ntuple_type_hints', 'check_fun_type_hints', + # fs utils + 'chdir', 'safe_rmpath', 'create_py_exe', 'create_c_exe', 'get_testfn', + # os + 'get_winver', 'kernel_version', 'is_busybox', + # sync primitives + 'call_until', 'wait_for_pid', 'wait_for_file', 'wait_for_file_subproc', + # network + 'check_net_address', 'filter_proc_net_connections', + 'get_free_port', 'bind_socket', 'bind_unix_socket', 'tcp_socketpair', + 'unix_socketpair', 'create_sockets', + # compat + 'reload_module', 'import_module_by_path', + # others + 'warn', 'copyload_shared_lib', 'is_namedtuple' +] +# fmt: on + + +# =================================================================== +# --- constants +# =================================================================== + +# --- platforms + +PYPY = '__pypy__' in sys.builtin_module_names +FREE_THREADED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) +# whether we're running this test suite on a Continuous Integration service +GITHUB_ACTIONS = 'GITHUB_ACTIONS' in os.environ or 'CIBUILDWHEEL' in os.environ +CI_TESTING = GITHUB_ACTIONS +COVERAGE = 'COVERAGE_RUN' in os.environ +PYTEST_PARALLEL = "PYTEST_XDIST_WORKER" in os.environ # `make test-parallel` +# apparently they're the same +AARCH64 = platform.machine().lower() in {"aarch64", "arm64"} +RISCV64 = platform.machine() == "riscv64" + + +@functools.lru_cache +def macos_version(): + version_str = platform.mac_ver()[0] + version = tuple(map(int, version_str.split(".")[:2])) + if version == (10, 16): + # When built against an older macOS SDK, Python will report + # macOS 10.16 instead of the real version. + version_str = subprocess.check_output( + [ + sys.executable, + "-sS", + "-c", + "import platform; print(platform.mac_ver()[0])", + ], + env={"SYSTEM_VERSION_COMPAT": "0"}, + universal_newlines=True, + ) + version = tuple(map(int, version_str.split(".")[:2])) + return version + + +if MACOS: + MACOS_11PLUS = macos_version() > (10, 15) + MACOS_12PLUS = macos_version() >= (12, 0) +else: + MACOS_11PLUS = False + MACOS_12PLUS = False + + +# --- configurable defaults + +# how many times retry_on_failure() decorator will retry +NO_RETRIES = 10 +# bytes tolerance for system-wide related tests +TOLERANCE_SYS_MEM = 5 * 1024 * 1024 # 5MB +TOLERANCE_DISK_USAGE = 10 * 1024 * 1024 # 10MB +# the timeout used in functions which have to wait +GLOBAL_TIMEOUT = 5 +# be more tolerant if we're on CI in order to avoid false positives +if CI_TESTING: + NO_RETRIES *= 3 + GLOBAL_TIMEOUT *= 3 + TOLERANCE_SYS_MEM *= 4 + TOLERANCE_DISK_USAGE *= 3 + +# --- file names + +# Disambiguate TESTFN with PID for parallel testing. +TESTFN_PREFIX = f"@psutil-{os.getpid()}-" +UNICODE_SUFFIX = "-ƒőő" +# An invalid unicode string. +INVALID_UNICODE_SUFFIX = b"f\xc0\x80".decode('utf8', 'surrogateescape') +ASCII_FS = sys.getfilesystemencoding().lower() in {"ascii", "us-ascii"} + +# --- paths + +ROOT_DIR = os.environ.get("PSUTIL_ROOT_DIR") or str( + pathlib.Path(__file__).resolve().parent.parent +) + +# --- support + +HAS_HEAP_INFO = hasattr(psutil, "heap_info") +HAS_NET_CONNECTIONS_UNIX = POSIX and not SUNOS +HAS_NET_IO_COUNTERS = hasattr(psutil, "net_io_counters") +HAS_SENSORS_BATTERY = hasattr(psutil, "sensors_battery") +HAS_SENSORS_FANS = hasattr(psutil, "sensors_fans") +HAS_SENSORS_TEMPERATURES = hasattr(psutil, "sensors_temperatures") + +HAS_PROC_CPU_AFFINITY = hasattr(psutil.Process, "cpu_affinity") +HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num") +HAS_PROC_ENVIRON = hasattr(psutil.Process, "environ") +HAS_PROC_IO_COUNTERS = hasattr(psutil.Process, "io_counters") +HAS_PROC_IONICE = hasattr(psutil.Process, "ionice") +HAS_PROC_MEMORY_EXTRAS = hasattr(psutil.Process, "memory_extras") +HAS_PROC_MEMORY_FOOTPRINT = hasattr(psutil.Process, "memory_footprint") +HAS_PROC_MEMORY_MAPS = hasattr(psutil.Process, "memory_maps") +HAS_PROC_RLIMIT = hasattr(psutil.Process, "rlimit") +HAS_PROC_THREADS = hasattr(psutil.Process, "threads") +HAS_PROC_OPEN_FILES_PATH = not (NETBSD or OPENBSD) + +SKIP_SYSCONS = (MACOS or AIX) and os.getuid() != 0 + +try: + HAS_BATTERY = HAS_SENSORS_BATTERY and bool(psutil.sensors_battery()) +except Exception: # noqa: BLE001 + atexit.register(functools.partial(print, traceback.format_exc())) + HAS_BATTERY = False +try: + HAS_CPU_FREQ = hasattr(psutil, "cpu_freq") and bool(psutil.cpu_freq()) +except Exception: # noqa: BLE001 + atexit.register(functools.partial(print, traceback.format_exc())) + HAS_CPU_FREQ = False + + +# --- misc + + +def _get_py_exe(): + def attempt(exe): + try: + subprocess.check_call( + [exe, "-V"], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + except subprocess.CalledProcessError: + return None + else: + return exe + + env = os.environ.copy() + + # Subprocesses (scripts, pyrun(), ...) get sys.path[0] set to the + # script's directory, so by default they import whatever psutil is + # installed instead of the one we're testing. Point them at ours. + # Derived from psutil.__file__ and not from ROOT_DIR because when + # testing wheels the source tree next to us has no C extension. + psutil_path = str(pathlib.Path(psutil.__file__).resolve().parent.parent) + paths = [psutil_path] + # Handle venvs. + if sys.prefix != sys.base_prefix: + paths.append(sysconfig.get_paths()["purelib"]) + env["PYTHONPATH"] = os.pathsep.join( + filter(None, [*paths, env.get("PYTHONPATH")]) + ) + + if PYPY and POSIX: + libdir = os.path.dirname(os.path.realpath(sys.executable)) + env["LD_LIBRARY_PATH"] = os.pathsep.join( + filter(None, [libdir, env.get("LD_LIBRARY_PATH")]) + ) + + # On Windows virtual environments use a venv launcher startup + # process. This does not play well when counting spawned processes, + # or when relying on the PID of the spawned process to do some + # checks, e.g. connections check per PID. Let's use the base python + # in this case. + base = getattr(sys, "_base_executable", None) + if WINDOWS and base is not None: + # We need to set __PYVENV_LAUNCHER__ to sys.executable for the + # base python executable to know about the environment. + env["__PYVENV_LAUNCHER__"] = sys.executable + return base, env + elif GITHUB_ACTIONS: + return sys.executable, env + elif MACOS: + exe = ( + attempt(sys.executable) + or attempt(os.path.realpath(sys.executable)) + or attempt( + shutil.which("python{}.{}".format(*sys.version_info[:2])) + ) + or attempt(psutil.Process().exe()) + ) + if not exe: + raise ValueError("can't find python exe real abspath") + return exe, env + else: + exe = os.path.realpath(sys.executable) + assert os.path.exists(exe), exe + return exe, env + + +PYTHON_EXE, PYTHON_EXE_ENV = _get_py_exe() +DEVNULL = open(os.devnull, 'r+') # noqa: SIM115 +atexit.register(DEVNULL.close) + +VALID_PROC_STATUSES = [ + getattr(psutil, x) for x in dir(psutil) if x.startswith('STATUS_') +] +AF_UNIX = getattr(socket, "AF_UNIX", object()) + +_subprocesses_started = set() +_pids_started = set() + + +# =================================================================== +# --- threads +# =================================================================== + + +class ThreadTask(threading.Thread): + """A thread task which does nothing expect staying alive.""" + + def __init__(self): + super().__init__() + self._running = False + self._interval = 0.001 + self._flag = threading.Event() + + def __repr__(self): + name = self.__class__.__name__ + return f"<{name} running={self._running} at {id(self):#x}>" + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args, **kwargs): + self.stop() + + def start(self): + """Start thread and keep it running until an explicit + stop() request. Polls for shutdown every 'timeout' seconds. + """ + if self._running: + raise ValueError("already started") + threading.Thread.start(self) + self._flag.wait() + + def run(self): + self._running = True + self._flag.set() + while self._running: + time.sleep(self._interval) + + def stop(self): + """Stop thread execution and and waits until it is stopped.""" + if not self._running: + raise ValueError("already stopped") + self._running = False + self.join() + + +# =================================================================== +# --- subprocesses +# =================================================================== + + +def _reap_children_on_err(fun): + @functools.wraps(fun) + def wrapper(*args, **kwargs): + try: + return fun(*args, **kwargs) + except Exception: + reap_children() + raise + + return wrapper + + +@_reap_children_on_err +def spawn_subproc(cmd=None, **kwds): + """Create a python subprocess which does nothing for some secs and + return it as a subprocess.Popen instance. + If "cmd" is specified that is used instead of python. + By default stdin and stdout are redirected to /dev/null. + It also attempts to make sure the process is in a reasonably + initialized state. + The process is registered for cleanup on reap_children(). + """ + kwds.setdefault("stdin", DEVNULL) + kwds.setdefault("stdout", DEVNULL) + kwds.setdefault("cwd", os.getcwd()) + kwds.setdefault("env", PYTHON_EXE_ENV) + if WINDOWS: + # Prevents the subprocess to open error dialogs. This will also + # cause stderr to be suppressed, which is suboptimal in order + # to debug broken tests. + # CREATE_NO_WINDOW = 0x8000000 + # kwds.setdefault("creationflags", CREATE_NO_WINDOW) + + # New: hopefully this should achieve the same and not suppress + # stderr. + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + startupinfo.wShowWindow = subprocess.SW_HIDE + kwds.setdefault("startupinfo", startupinfo) + + if cmd is None: + testfn = get_testfn(dir=os.getcwd()) + try: + safe_rmpath(testfn) + pyline = ( + "import time;" + f"open(r'{testfn}', 'w').close();" + "[time.sleep(0.1) for x in range(100)];" # 10 secs + ) + cmd = [PYTHON_EXE, "-c", pyline] + kwds.setdefault("stderr", subprocess.PIPE) + sproc = subprocess.Popen(cmd, **kwds) + _subprocesses_started.add(sproc) + wait_for_file_subproc(testfn, sproc, delete=True, empty=True) + finally: + safe_rmpath(testfn) + else: + sproc = subprocess.Popen(cmd, **kwds) + _subprocesses_started.add(sproc) + wait_for_pid(sproc.pid) + _wait_for_cmdline(sproc.pid) + return sproc + + +@_reap_children_on_err +def spawn_children_pair(): + """Create a subprocess which creates another one as in: + A (us) -> B (child) -> C (grandchild). + Return a (child, grandchild) tuple. + The 2 processes are fully initialized and will live for 60 secs + and are registered for cleanup on reap_children(). + """ + tfile = None + testfn = get_testfn(dir=os.getcwd()) + try: + s = textwrap.dedent(f"""\ + import subprocess, os, sys, time + s = "import os, time;" + s += "f = open('{os.path.basename(testfn)}', 'w');" + s += "f.write(str(os.getpid()));" + s += "f.close();" + s += "[time.sleep(0.1) for x in range(100 * 6)];" + p = subprocess.Popen([r'{PYTHON_EXE}', '-c', s]) + p.wait() + """) + # On Windows if we create a subprocess with CREATE_NO_WINDOW flag + # set (which is the default) a "conhost.exe" extra process will be + # spawned as a child. We don't want that. + if WINDOWS: + subp, tfile = pyrun(s, creationflags=0, stderr=subprocess.PIPE) + else: + subp, tfile = pyrun(s, stderr=subprocess.PIPE) + child = psutil.Process(subp.pid) + grandchild_pid = int( + wait_for_file_subproc(testfn, subp, delete=True, empty=False) + ) + _pids_started.add(grandchild_pid) + grandchild = psutil.Process(grandchild_pid) + return (child, grandchild) + finally: + safe_rmpath(testfn) + if tfile is not None: + safe_rmpath(tfile) + + +def spawn_zombie(): + """Create a zombie process and return a (parent, zombie) process tuple. + In order to kill the zombie parent must be terminate()d first, then + zombie must be wait()ed on. + """ + assert psutil.POSIX + unix_file = get_testfn() + src = textwrap.dedent(f"""\ + import os, socket, time + child_pid = os.fork() + if child_pid == 0: + os._exit(0) + else: + with socket.socket(socket.AF_UNIX) as s: + s.connect('{unix_file}') + s.sendall(bytes(str(child_pid), 'ascii')) + time.sleep(3000) + """) + tfile = None + sock = bind_unix_socket(unix_file) + try: + sock.settimeout(GLOBAL_TIMEOUT) + parent, tfile = pyrun(src) + conn, _ = sock.accept() + try: + select.select([conn.fileno()], [], [], GLOBAL_TIMEOUT) + zpid = int(conn.recv(1024)) + _pids_started.add(zpid) + zombie = psutil.Process(zpid) + call_until(lambda: zombie.status() == psutil.STATUS_ZOMBIE) + return (parent, zombie) + finally: + conn.close() + finally: + sock.close() + safe_rmpath(unix_file) + if tfile is not None: + safe_rmpath(tfile) + + +@_reap_children_on_err +def pyrun(src, **kwds): + """Run python 'src' code string in a separate interpreter. + Returns a subprocess.Popen instance and the test file where the source + code was written. + """ + kwds.setdefault("stdout", None) + kwds.setdefault("stderr", None) + srcfile = get_testfn() + try: + with open(srcfile, "w") as f: + f.write(src) + subp = spawn_subproc([PYTHON_EXE, f.name], **kwds) + wait_for_pid(subp.pid) + return (subp, srcfile) + except Exception: + safe_rmpath(srcfile) + raise + + +@_reap_children_on_err +def sh(cmd, **kwds): + """Run cmd in a subprocess and return its output. + raises RuntimeError on error. + """ + # Prevents subprocess to open error dialogs in case of error. + flags = 0x8000000 if WINDOWS else 0 + kwds.setdefault("stdout", subprocess.PIPE) + kwds.setdefault("stderr", subprocess.PIPE) + kwds.setdefault("universal_newlines", True) + kwds.setdefault("encoding", ENCODING) + kwds.setdefault("errors", ENCODING_ERRS) + kwds.setdefault("creationflags", flags) + if isinstance(cmd, str): + cmd = shlex.split(cmd) + p = subprocess.Popen(cmd, **kwds) + _subprocesses_started.add(p) + stdout, stderr = p.communicate(timeout=GLOBAL_TIMEOUT) + if p.returncode != 0: + raise RuntimeError(stdout + stderr) + if stderr: + warn(stderr) + if stdout.endswith('\n'): + stdout = stdout[:-1] + return stdout + + +def terminate(proc_or_pid, sig=signal.SIGTERM, wait_timeout=GLOBAL_TIMEOUT): + """Terminate a process and wait() for it. + Process can be a PID or an instance of psutil.Process(), + subprocess.Popen() or psutil.Popen(). + If it's a subprocess.Popen() or psutil.Popen() instance also closes + its stdin / stdout / stderr fds. + PID is wait()ed even if the process is already gone (kills zombies). + Does nothing if the process does not exist. + Return process exit status. + """ + + def wait(proc, timeout): + proc.wait(timeout) + if WINDOWS and isinstance(proc, subprocess.Popen): + # Otherwise PID may still hang around. + try: + return psutil.Process(proc.pid).wait(timeout) + except psutil.NoSuchProcess: + pass + + def sendsig(proc, sig): + # XXX: otherwise the build hangs for some reason. + if MACOS and GITHUB_ACTIONS: + sig = signal.SIGKILL + # If the process received SIGSTOP, SIGCONT is necessary first, + # otherwise SIGTERM won't work. + if POSIX and sig != signal.SIGKILL: + proc.send_signal(signal.SIGCONT) + proc.send_signal(sig) + + def term_subprocess_proc(proc, timeout): + try: + sendsig(proc, sig) + except ProcessLookupError: + pass + except OSError as err: + if WINDOWS and err.winerror == 6: # "invalid handle" + pass + raise + return wait(proc, timeout) + + def term_psutil_proc(proc, timeout): + try: + sendsig(proc, sig) + except psutil.NoSuchProcess: + pass + return wait(proc, timeout) + + def term_pid(pid, timeout): + try: + proc = psutil.Process(pid) + except psutil.NoSuchProcess: + # Needed to kill zombies. + if POSIX: + return wait_pid(pid, timeout) + else: + return term_psutil_proc(proc, timeout) + + def flush_popen(proc): + if proc.stdout: + proc.stdout.close() + if proc.stderr: + proc.stderr.close() + # Flushing a BufferedWriter may raise an error. + if proc.stdin: + proc.stdin.close() + + p = proc_or_pid + try: + if isinstance(p, int): + return term_pid(p, wait_timeout) + elif isinstance(p, (psutil.Process, psutil.Popen)): + return term_psutil_proc(p, wait_timeout) + elif isinstance(p, subprocess.Popen): + return term_subprocess_proc(p, wait_timeout) + else: + raise TypeError(f"wrong type {p!r}") + finally: + if isinstance(p, (subprocess.Popen, psutil.Popen)): + flush_popen(p) + pid = p if isinstance(p, int) else p.pid + assert not psutil.pid_exists(pid), pid + + +def filter_alien_children(procs): + """On Windows CI the runner agent (provjobd.exe) sporadically + spawns wsl.exe, conhost.exe, etc. When their parent dies the PPID + is left dangling (Windows never clears it), and if that PID gets + reused by us they show up as our children. + """ + if not (WINDOWS and CI_TESTING): + return procs + names = {"wsl.exe", "conhost.exe"} + aliens = { + x.pid + for x in psutil.process_iter(["name"]) + if (x.name() or "").lower() in names + } + return [x for x in procs if x.pid not in aliens] + + +def reap_children(recursive=False): + """Terminate and wait() any subprocess started by this test suite + and any children currently running, ensuring that no processes stick + around to hog resources. + If recursive is True it also tries to terminate and wait() + all grandchildren started by this process. + """ + # Get the children here before terminating them, as in case of + # recursive=True we don't want to lose the intermediate reference + # pointing to the grandchildren. + children = psutil.Process().children(recursive=recursive) + # children() lists them top-down; reverse so descendants die before + # their parents (avoids orphaning grandchildren). + children.reverse() + + # Terminate subprocess.Popen. + while _subprocesses_started: + subp = _subprocesses_started.pop() + terminate(subp) + + # Collect started pids. + while _pids_started: + pid = _pids_started.pop() + terminate(pid) + + # Terminate children. + if children: + timeout = 3 + for p in children: + try: + terminate(p, wait_timeout=timeout) + except psutil.TimeoutExpired: + warn(f"{p!r} didn't terminate within {timeout} secs") + _, alive = psutil.wait_procs(children, timeout=GLOBAL_TIMEOUT) + for p in alive: + warn(f"couldn't terminate process {p!r}; attempting kill()") + terminate(p, sig=signal.SIGKILL) + + +# =================================================================== +# --- OS +# =================================================================== + + +def kernel_version(): + """Return a tuple such as (2, 6, 36).""" + if not POSIX: + raise NotImplementedError("not POSIX") + s = "" + uname = os.uname()[2] + for c in uname: + if c.isdigit() or c == '.': + s += c + else: + break + if not s: + raise ValueError(f"can't parse {uname!r}") + minor = 0 + micro = 0 + nums = s.split('.') + major = int(nums[0]) + if len(nums) >= 2: + minor = int(nums[1]) + if len(nums) >= 3: + micro = int(nums[2]) + return (major, minor, micro) + + +def get_winver(): + if not WINDOWS: + raise NotImplementedError("not WINDOWS") + wv = sys.getwindowsversion() + sp = wv.service_pack_major or 0 + return (wv[0], wv[1], sp) + + +@functools.lru_cache +def is_busybox(cmd): + """Whether cmd is provided by busybox / Alpine Linux.""" + path = shutil.which(cmd) + if path is None: + return False + return os.path.basename(os.path.realpath(path)) == "busybox" + + +# =================================================================== +# --- sync primitives +# =================================================================== + + +class retry: + """A retry decorator.""" + + def __init__( + self, + exception=Exception, + timeout=None, + retries=None, + interval=0.001, + logfun=None, + ): + if timeout and retries: + raise ValueError("timeout and retries args are mutually exclusive") + self.exception = exception + self.timeout = timeout + self.retries = retries + self.interval = interval + self.logfun = logfun + + def __iter__(self): + if self.timeout: + # time.monotonic(): the BSD CI VMs step the system clock + # (NTP), which would expire a time.time() deadline early. + stop_at = time.monotonic() + self.timeout + while time.monotonic() < stop_at: + yield + elif self.retries: + for _ in range(self.retries): + yield + else: + while True: + yield + + def sleep(self): + if self.interval is not None: + time.sleep(self.interval) + + def __call__(self, fun): + @functools.wraps(fun) + def wrapper(*args, **kwargs): + exc = None + for _ in self: + try: + return fun(*args, **kwargs) + except self.exception as _: + exc = _ + if self.logfun is not None: + self.logfun(exc) + self.sleep() + continue + + raise exc + + # This way the user of the decorated function can change config + # parameters. + wrapper.decorator = self + return wrapper + + +@retry( + exception=psutil.NoSuchProcess, + logfun=None, + timeout=GLOBAL_TIMEOUT, + interval=0.001, +) +def wait_for_pid(pid): + """Wait for pid to show up in the process list then return. + Used in the test suite to give time the sub process to initialize. + """ + if pid not in psutil.pids(): + raise psutil.NoSuchProcess(pid) + psutil.Process(pid) + + +@retry( + exception=(FileNotFoundError, AssertionError), + logfun=None, + timeout=GLOBAL_TIMEOUT, + interval=0.001, +) +def wait_for_file(fname, delete=True, empty=False): + """Wait for a file to be written on disk with some content.""" + with open(fname, "rb") as f: + data = f.read() + if not empty: + assert data + if delete: + safe_rmpath(fname) + return data + + +def wait_for_file_subproc(fname, sproc, delete=True, empty=False): + """Wait for a file to be written on disk, which is supposed to be + written by a subprocess. + """ + try: + return wait_for_file(fname, delete=delete, empty=empty) + except FileNotFoundError as err: + ret = sproc.poll() + if ret is None: + raise + stderr = sproc.stderr.read() if sproc.stderr else b"" + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + msg = f"subprocess died (exit {ret}):\n{stderr}" + raise RuntimeError(msg) from err + + +@retry( + exception=(AssertionError, psutil.AccessDenied), + logfun=None, + timeout=GLOBAL_TIMEOUT, + interval=0.001, +) +def _wait_for_cmdline(pid): + # Popen() returns before the kernel publishes argv, so for a moment + # cmdline reads back empty on Linux, and raises AccessDenied on + # macOS, where sysctl(KERN_PROCARGS2) fails with EINVAL. + try: + assert psutil.Process(pid).cmdline() + except (psutil.NoSuchProcess, psutil.ZombieProcess): + return + + +@retry( + exception=(AssertionError, pytest.fail.Exception), + logfun=None, + timeout=GLOBAL_TIMEOUT, + interval=0.001, +) +def call_until(fun): + """Keep calling function until it evaluates to True.""" + ret = fun() + assert ret + return ret + + +# =================================================================== +# --- fs +# =================================================================== + + +def safe_rmpath(path): + """Convenience function for removing temporary test files or dirs.""" + + def retry_fun(fun): + # On Windows it could happen that the file or directory has + # open handles or references preventing the delete operation + # to succeed immediately, so we retry for a while. See: + # https://bugs.python.org/issue33240 + stop_at = time.monotonic() + GLOBAL_TIMEOUT + while time.monotonic() < stop_at: + try: + return fun() + except FileNotFoundError: + pass + except OSError as _: + err = _ + warn(f"ignoring {err}") + time.sleep(0.01) + raise err + + try: + st = os.stat(path) + if stat.S_ISDIR(st.st_mode): + fun = functools.partial(shutil.rmtree, path) + else: + fun = functools.partial(os.remove, path) + if POSIX: + fun() + else: + retry_fun(fun) + except FileNotFoundError: + pass + + +def safe_mkdir(dir): + """Convenience function for creating a directory.""" + try: + os.mkdir(dir) + except FileExistsError: + pass + + +@contextlib.contextmanager +def chdir(dirname): + """Context manager which temporarily changes the current directory.""" + curdir = os.getcwd() + try: + os.chdir(dirname) + yield + finally: + os.chdir(curdir) + + +def create_py_exe(path): + """Create a Python executable file in the given location.""" + assert not os.path.exists(path), path + atexit.register(safe_rmpath, path) + shutil.copyfile(PYTHON_EXE, path) + if POSIX: + st = os.stat(path) + os.chmod(path, st.st_mode | stat.S_IEXEC) + return path + + +def create_c_exe(path, c_code=None): + """Create a compiled C executable in the given location.""" + assert not os.path.exists(path), path + if not shutil.which("gcc"): + return pytest.skip("gcc is not installed") + if c_code is None: + c_code = textwrap.dedent(""" + #include + int main() { + pause(); + return 1; + } + """) + else: + assert isinstance(c_code, str), c_code + + atexit.register(safe_rmpath, path) + with open(get_testfn(suffix='.c'), "w") as f: + f.write(c_code) + try: + subprocess.check_call(["gcc", f.name, "-o", path]) + finally: + safe_rmpath(f.name) + return path + + +def get_testfn(suffix="", dir=None): + """Return an absolute pathname of a file or dir that did not + exist at the time this call is made. Also schedule it for safe + deletion at interpreter exit. It's technically racy but probably + not really due to the time variant. + """ + name = tempfile.mktemp(prefix=TESTFN_PREFIX, suffix=suffix, dir=dir) + path = os.path.realpath(name) # needed for OSX + atexit.register(safe_rmpath, path) + return path + + +# =================================================================== +# --- testing +# =================================================================== + +# `@serial` decorator: put all marked tests on the same xdist worker, +# so they don't run concurrently in the same process. Needed by tests +# that share the same system-wide resource (e.g. a socket) and must not +# overlap. +# - net_connections() / Process.net_connections() that compare vs +# `ss`, `netstat`, etc. +# - the socket-opening helpers: create_sockets(), bind_socket(), +# tcp_socketpair(), unix_socketpair(), bind_unix_socket() +serial = pytest.mark.xdist_group(name="serial") + +# `@isolated` decorator: these tests are skipped under xdist and run in +# a second, separate pytest run (`-m isolated`) that uses a single +# process. They need a quiet, non-xdist environment, because +# they measure noisy per-process or system counters. +# - CPU counters compared vs getrusage / vmstat / WMI: cpu_stats(), +# Process.num_ctx_switches(), Process.page_faults() +# - per-process counts other tests can move: Process.num_threads(), +# Process.num_fds(), Process.threads(), heap_info() +isolated = pytest.mark.isolated + +skipif = pytest.mark.skipif + + +def requires_cli(cmd): + """Skip test if CLI command is not available.""" + + def outer(fun): + @functools.wraps(fun) + def inner(*args, **kwargs): + if not shutil.which(cmd): + pytest.skip(f"{cmd} cmd not available") + return fun(*args, **kwargs) + + return inner + + return outer + + +class PsutilTestCase(unittest.TestCase): + """Test class providing auto-cleanup wrappers on top of process + test utilities. All test classes should derive from this one, even + if we use pytest. + """ + + # Print a full path representation of the single unit test being + # run, similar to pytest output. Used only when running tests with + # the unittest runner. + def __str__(self): + fqmod = self.__class__.__module__ + if not fqmod.startswith('psutil.'): + fqmod = 'tests.' + fqmod + return "{}.{}.{}".format( + fqmod, + self.__class__.__name__, + self._testMethodName, + ) + + def get_testfn(self, suffix="", dir=None): + fname = get_testfn(suffix=suffix, dir=dir) + self.addCleanup(safe_rmpath, fname) + return fname + + def spawn_subproc(self, *args, **kwds): + sproc = spawn_subproc(*args, **kwds) + self.addCleanup(terminate, sproc) + return sproc + + def spawn_psproc(self, *args, **kwargs): + sproc = self.spawn_subproc(*args, **kwargs) + try: + return psutil.Process(sproc.pid) + except psutil.NoSuchProcess: + self.assert_pid_gone(sproc.pid) + raise + + def spawn_children_pair(self): + child1, child2 = spawn_children_pair() + self.addCleanup(terminate, child2) + self.addCleanup(terminate, child1) # executed first + return (child1, child2) + + def spawn_zombie(self): + parent, zombie = spawn_zombie() + self.addCleanup(terminate, zombie) + self.addCleanup(terminate, parent) # executed first + return (parent, zombie) + + def pyrun(self, *args, **kwds): + sproc, srcfile = pyrun(*args, **kwds) + self.addCleanup(safe_rmpath, srcfile) + self.addCleanup(terminate, sproc) # executed first + return sproc + + def _check_proc_exc(self, proc, exc): + assert isinstance(exc, psutil.Error) + assert exc.pid == proc.pid + assert exc.name == proc._name + if exc.name: + assert exc.name + if isinstance(exc, psutil.ZombieProcess): + assert exc.ppid == proc._ppid + if exc.ppid is not None: + assert exc.ppid >= 0 + str(exc) + repr(exc) + + def assert_pid_gone(self, pid): + try: + proc = psutil.Process(pid) + except psutil.ZombieProcess: + raise AssertionError("wasn't supposed to raise ZombieProcess") + except psutil.NoSuchProcess as exc: + assert exc.pid == pid # noqa: PT017 + assert exc.name is None # noqa: PT017 + else: + raise AssertionError(f"did not raise NoSuchProcess ({proc})") + + assert not psutil.pid_exists(pid), pid + assert pid not in psutil.pids() + assert pid not in [x.pid for x in psutil.process_iter()] + + def assert_proc_gone(self, proc): + self.assert_pid_gone(proc.pid) + ns = process_namespace(proc) + for fun, name in ns.iter(ns.all, clear_cache=True): + with self.subTest(proc=str(proc), name=name): + try: + ret = fun() + except psutil.ZombieProcess: + raise + except psutil.NoSuchProcess as exc: + self._check_proc_exc(proc, exc) + else: + msg = ( + f"Process.{name}() didn't raise NSP and returned" + f" {ret!r}" + ) + raise AssertionError(msg) + proc.wait(timeout=0) # assert not raise TimeoutExpired + + def assert_proc_zombie(self, proc): + def assert_in_pids(proc): + if MACOS: + # Even ps does not show zombie PIDs for some reason. Weird... + return + assert proc.pid in psutil.pids() + assert proc.pid in [x.pid for x in psutil.process_iter()] + psutil._pmap = {} + assert proc.pid in [x.pid for x in psutil.process_iter()] + + # A zombie process should always be instantiable. + clone = psutil.Process(proc.pid) + # Cloned zombie on Open/NetBSD/illumos/Solaris has null creation + # time, see: + # https://github.com/giampaolo/psutil/issues/2287 + # https://github.com/giampaolo/psutil/issues/2593 + assert proc == clone + if not (OPENBSD or NETBSD or SUNOS): + assert hash(proc) == hash(clone) + # Its status always be querable. + assert proc.status() == psutil.STATUS_ZOMBIE + # It should be considered 'running'. + assert proc.is_running() + assert psutil.pid_exists(proc.pid) + # as_dict() shouldn't crash. + proc.as_dict() + # It should show up in pids() and process_iter(). + assert_in_pids(proc) + # Call all methods. + ns = process_namespace(proc) + for fun, name in ns.iter(ns.all, clear_cache=True): + with self.subTest(proc=str(proc), name=name): + try: + fun() + except (psutil.ZombieProcess, psutil.AccessDenied) as exc: + self._check_proc_exc(proc, exc) + if LINUX: + # https://github.com/giampaolo/psutil/pull/2288 + with pytest.raises(psutil.ZombieProcess) as cm: + proc.cmdline() + self._check_proc_exc(proc, cm.value) + with pytest.raises(psutil.ZombieProcess) as cm: + proc.exe() + self._check_proc_exc(proc, cm.value) + with pytest.raises(psutil.ZombieProcess) as cm: + proc.memory_maps() + self._check_proc_exc(proc, cm.value) + # Zombie cannot be signaled or terminated. Another user's zombie + # can't be signaled at all, so only try on our own. + if not POSIX or proc.uids().real == os.getuid(): + proc.suspend() + proc.resume() + proc.terminate() + proc.kill() + assert proc.is_running() + assert psutil.pid_exists(proc.pid) + assert_in_pids(proc) + + # Its parent should 'see' it (edit: not true on BSD and MACOS). + # descendants = [x.pid for x in psutil.Process().children( + # recursive=True)] + # assert proc.pid in descendants + + # __eq__ can't be relied upon because creation time may not be + # querable. + # assert proc == psutil.Process(proc.pid) + + # XXX should we also assume ppid() to be usable? Note: this + # would be an important use case as the only way to get + # rid of a zombie is to kill its parent. + # assert proc == ppid(), os.getpid() + + def check_proc_memory(self, nt): + # Check the ntuple returned by Process.memory_*() methods. + check_ntuple_type_hints(nt) + for value in nt: + assert isinstance(value, int) + assert value >= 0 + if hasattr(nt, "peak_rss") and hasattr(nt, "rss"): + if BSD and nt.peak_rss == 0: + pass # kernel threads don't have rusage tracking + else: + # VmHWM (from /proc/pid/status) and ru_maxrss both + # track peak RSS but are synced independently. Allow 5% + # tolerance. + diff = nt.rss - nt.peak_rss + assert diff <= nt.rss * 0.05 + + +def is_win_secure_system_proc(pid): + # see: https://github.com/giampaolo/psutil/issues/2338 + @functools.lru_cache + def get_procs(): + ret = {} + out = sh("tasklist.exe /NH /FO csv") + for line in out.splitlines()[1:]: + bits = [x.replace('"', "") for x in line.split(",")] + name, pid = bits[0], int(bits[1]) + ret[pid] = name + return ret + + try: + return get_procs()[pid] == "Secure System" + except KeyError: + return False + + +def _get_eligible_cpu(): + p = psutil.Process() + if hasattr(p, "cpu_num"): + return p.cpu_num() + elif hasattr(p, "cpu_affinity"): + return random.choice(p.cpu_affinity()) + return 0 + + +class process_namespace: + """A container that lists all Process class method names + some + reasonable parameters to be called with. Utility methods (parent(), + children(), ...) are excluded. + + >>> ns = process_namespace(psutil.Process()) + >>> for fun, name in ns.iter(ns.getters): + ... fun() + """ + + utils = [('cpu_percent', (), {}), ('memory_percent', (), {})] + + ignored = [ + ('as_dict', (), {}), + ('attrs', (), {}), + ('children', (), {'recursive': True}), + ('connections', (), {}), # deprecated + ('info', (), {}), + ('is_running', (), {}), + ('memory_full_info', (), {}), # deprecated + ('oneshot', (), {}), + ('parent', (), {}), + ('parents', (), {}), + ('pid', (), {}), + ('wait', (0,), {}), + ] + + getters = [ + ('cmdline', (), {}), + ('cpu_times', (), {}), + ('create_time', (), {}), + ('cwd', (), {}), + ('exe', (), {}), + ('memory_info', (), {}), + ('name', (), {}), + ('net_connections', (), {'kind': 'all'}), + ('nice', (), {}), + ('num_ctx_switches', (), {}), + ('num_threads', (), {}), + ('open_files', (), {}), + ('page_faults', (), {}), + ('ppid', (), {}), + ('status', (), {}), + ('threads', (), {}), + ('username', (), {}), + ] + if POSIX: + getters += [('uids', (), {})] + getters += [('gids', (), {})] + getters += [('terminal', (), {})] + getters += [('num_fds', (), {})] + if HAS_PROC_IO_COUNTERS: + getters += [('io_counters', (), {})] + if HAS_PROC_IONICE: + getters += [('ionice', (), {})] + if HAS_PROC_RLIMIT: + getters += [('rlimit', (psutil.RLIMIT_NOFILE,), {})] + if HAS_PROC_CPU_AFFINITY: + getters += [('cpu_affinity', (), {})] + if HAS_PROC_CPU_NUM: + getters += [('cpu_num', (), {})] + if HAS_PROC_ENVIRON: + getters += [('environ', (), {})] + if WINDOWS: + getters += [('num_handles', (), {})] + if HAS_PROC_MEMORY_EXTRAS: + getters += [('memory_extras', (), {})] + if HAS_PROC_MEMORY_FOOTPRINT: + getters += [('memory_footprint', (), {})] + if HAS_PROC_MEMORY_MAPS: + getters += [('memory_maps', (), {'grouped': True})] + getters += [('memory_maps', (), {'grouped': False})] + + setters = [] + if POSIX: + setters += [('nice', (0,), {})] + else: + setters += [('nice', (psutil.NORMAL_PRIORITY_CLASS,), {})] + if HAS_PROC_RLIMIT: + setters += [('rlimit', (psutil.RLIMIT_NOFILE, (1024, 4096)), {})] + if HAS_PROC_IONICE: + if LINUX: + setters += [('ionice', (psutil.IOPRIO_CLASS_NONE, 0), {})] + else: + setters += [('ionice', (psutil.IOPRIO_NORMAL,), {})] + if HAS_PROC_CPU_AFFINITY: + setters += [('cpu_affinity', ([_get_eligible_cpu()],), {})] + + killers = [ + ('send_signal', (signal.SIGTERM,), {}), + ('suspend', (), {}), + ('resume', (), {}), + ('terminate', (), {}), + ('kill', (), {}), + ] + if WINDOWS: + killers += [('send_signal', (signal.CTRL_C_EVENT,), {})] + killers += [('send_signal', (signal.CTRL_BREAK_EVENT,), {})] + + all = utils + getters + setters + killers + + def __init__(self, proc): + self._proc = proc + + def iter(self, ls, clear_cache=True): + """Given a list of tuples yields a set of (fun, fun_name) tuples + in random order. + """ + ls = list(ls) + random.shuffle(ls) + for fun_name, args, kwds in ls: + if clear_cache: + self.clear_cache() + fun = getattr(self._proc, fun_name) + fun = functools.partial(fun, *args, **kwds) + yield (fun, fun_name) + + def clear_cache(self): + """Clear the cache of a Process instance.""" + self._proc._init(self._proc.pid, _ignore_nsp=True) + + @classmethod + def test_class_coverage(cls, test_class, ls): + """Given a TestCase instance and a list of tuples checks that + the class defines the required test method names. + """ + for fun_name, _, _ in ls: + meth_name = 'test_' + fun_name + if not hasattr(test_class, meth_name): + msg = ( + f"{test_class.__class__.__name__!r} class should define a" + f" {meth_name!r} method" + ) + raise AttributeError(msg) + + @classmethod + def test(cls): + this = {x[0] for x in cls.all} + ignored = {x[0] for x in cls.ignored} + klass = {x for x in dir(psutil.Process) if x[0] != '_'} + leftout = (this | ignored) ^ klass + if leftout: + raise ValueError(f"uncovered Process class names: {leftout!r}") + + +class system_namespace: + """A container that lists all the module-level, system-related APIs. + Utilities such as cpu_percent() are excluded. Usage: + + >>> ns = system_namespace + >>> for fun, name in ns.iter(ns.getters): + ... fun() + """ + + getters = [ + ('boot_time', (), {}), + ('cpu_count', (), {'logical': False}), + ('cpu_count', (), {'logical': True}), + ('cpu_stats', (), {}), + ('cpu_times', (), {'percpu': False}), + ('cpu_times', (), {'percpu': True}), + ('disk_io_counters', (), {'perdisk': False}), + ('disk_io_counters', (), {'perdisk': True}), + ('disk_partitions', (), {'all': False}), + ('disk_partitions', (), {'all': True}), + ('disk_usage', (os.getcwd(),), {}), + ('getloadavg', (), {}), + ('net_connections', (), {'kind': 'all'}), + ('net_if_addrs', (), {}), + ('net_if_stats', (), {}), + ('net_io_counters', (), {'pernic': False}), + ('net_io_counters', (), {'pernic': True}), + ('pid_exists', (os.getpid(),), {}), + ('pids', (), {}), + ('swap_memory', (), {}), + ('users', (), {}), + ('virtual_memory', (), {}), + ] + + if HAS_CPU_FREQ: + getters += [('cpu_freq', (), {'percpu': False})] + getters += [('cpu_freq', (), {'percpu': True})] + if HAS_SENSORS_TEMPERATURES: + getters += [('sensors_temperatures', (), {})] + if HAS_SENSORS_FANS: + getters += [('sensors_fans', (), {})] + if HAS_SENSORS_BATTERY: + getters += [('sensors_battery', (), {})] + if HAS_HEAP_INFO: + getters += [('heap_info', (), {})] + getters += [('heap_trim', (), {})] + + if WINDOWS: + getters += [('win_service_iter', (), {})] + getters += [('win_service_get', ('alg',), {})] + + ignored = [ + ('process_iter', (), {}), + ('wait_procs', ([psutil.Process()],), {}), + ('cpu_percent', (), {}), + ('cpu_times_percent', (), {}), + ] + + all = getters + + @staticmethod + def iter(ls): + """Given a list of tuples yields a set of (fun, fun_name) tuples + in random order. + """ + ls = list(ls) + random.shuffle(ls) + for fun_name, args, kwds in ls: + fun = getattr(psutil, fun_name) + fun = functools.partial(fun, *args, **kwds) + yield (fun, fun_name) + + test_class_coverage = process_namespace.test_class_coverage + + +def retry_on_failure(retries: "int | typing.Callable" = NO_RETRIES): + """Decorator which runs a test function and retries N times before + giving up and failing. + """ + + def decorator(test_method): + @functools.wraps(test_method) + def wrapper(self, *args, **kwargs): + err = None + for attempt in range(retries): + try: + return test_method(self, *args, **kwargs) + except (AssertionError, pytest.fail.Exception) as _: + err = _ + prefix = "\n" if attempt == 0 else "" + short_err = str(err).split("\n")[0] + print( # noqa: T201 + f"{prefix}{short_err}, retrying" + f" {attempt + 1}/{retries} ...", + file=sys.stderr, + ) + if hasattr(self, "tearDown"): + self.tearDown() + if hasattr(self, "teardown_method"): + self.teardown_method() + if hasattr(self, "setUp"): + self.setUp() + if hasattr(self, "setup_method"): + self.setup_method() + + raise err + + return wrapper + + # allow bare `@retry_on_failure` + if callable(retries): + return retry_on_failure()(retries) + assert retries > 1, retries + return decorator + + +def skip_on_access_denied(only_if: "bool | typing.Callable | None" = None): + """Decorator to Ignore AccessDenied exceptions.""" + + def decorator(fun): + @functools.wraps(fun) + def wrapper(*args, **kwargs): + try: + return fun(*args, **kwargs) + except psutil.AccessDenied: + if only_if is not None: + if not only_if: + raise + return pytest.skip("raises AccessDenied") + + return wrapper + + if callable(only_if): + return skip_on_access_denied()(only_if) + return decorator + + +def skip_on_not_implemented(only_if: "bool | typing.Callable | None" = None): + """Decorator to Ignore NotImplementedError exceptions.""" + + def decorator(fun): + @functools.wraps(fun) + def wrapper(*args, **kwargs): + try: + return fun(*args, **kwargs) + except NotImplementedError: + if only_if is not None: + if not only_if: + raise + msg = ( + f"{fun.__name__!r} was skipped because it raised" + " NotImplementedError" + ) + return pytest.skip(msg) + + return wrapper + + if callable(only_if): + return skip_on_not_implemented()(only_if) + return decorator + + +# =================================================================== +# --- network +# =================================================================== + + +# XXX: no longer used +def get_free_port(host='127.0.0.1'): + """Return an unused TCP port. Subject to race conditions.""" + with socket.socket() as sock: + sock.bind((host, 0)) + return sock.getsockname()[1] + + +def bind_socket(family=AF_INET, type=SOCK_STREAM, addr=None): + """Binds a generic socket.""" + if addr is None and family in {AF_INET, AF_INET6}: + addr = ("", 0) + sock = socket.socket(family, type) + try: + if os.name not in {'nt', 'cygwin'}: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(addr) + if type == socket.SOCK_STREAM: + sock.listen(5) + return sock + except Exception: + sock.close() + raise + + +def bind_unix_socket(name, type=socket.SOCK_STREAM): + """Bind a UNIX socket.""" + assert psutil.POSIX + assert not os.path.exists(name), name + sock = socket.socket(socket.AF_UNIX, type) + try: + sock.bind(name) + if type == socket.SOCK_STREAM: + sock.listen(5) + except Exception: + sock.close() + raise + return sock + + +def tcp_socketpair(family, addr=("", 0)): + """Build a pair of TCP sockets connected to each other. + Return a (server, client) tuple. + """ + with socket.create_server(addr, family=family, backlog=5) as ll: + ll.settimeout(GLOBAL_TIMEOUT) + addr = ll.getsockname() + c = socket.socket(family, SOCK_STREAM) + try: + c.connect(addr) + caddr = c.getsockname() + while True: + a, addr = ll.accept() + # check that we've got the correct client + if addr == caddr: + return (a, c) + a.close() + except OSError: + c.close() + raise + + +def unix_socketpair(name): + """Build a pair of UNIX sockets connected to each other through + the same UNIX file name. + Return a (server, client) tuple. + """ + assert psutil.POSIX + server = client = None + try: + server = bind_unix_socket(name, type=socket.SOCK_STREAM) + server.setblocking(0) + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.setblocking(0) + client.connect(name) + # new = server.accept() + except Exception: + if server is not None: + server.close() + if client is not None: + client.close() + raise + return (server, client) + + +@contextlib.contextmanager +def create_sockets(): + """Open as many socket families / types as possible.""" + socks = [] + fname1 = fname2 = None + try: + socks.extend(( + bind_socket(socket.AF_INET, socket.SOCK_STREAM), + bind_socket(socket.AF_INET, socket.SOCK_DGRAM), + )) + if supports_ipv6(): + socks.extend(( + bind_socket(socket.AF_INET6, socket.SOCK_STREAM), + bind_socket(socket.AF_INET6, socket.SOCK_DGRAM), + )) + if POSIX and HAS_NET_CONNECTIONS_UNIX: + fname1 = get_testfn() + fname2 = get_testfn() + s1, s2 = unix_socketpair(fname1) + s3 = bind_unix_socket(fname2, type=socket.SOCK_DGRAM) + for s in (s1, s2, s3): + socks.append(s) + yield socks + finally: + for s in socks: + s.close() + for fname in (fname1, fname2): + if fname is not None: + safe_rmpath(fname) + + +def check_net_address(addr, family): + """Check a net address validity. Supported families are IPv4, + IPv6 and MAC addresses. + """ + assert isinstance(family, enum.IntEnum), family + if family == socket.AF_INET: + octs = [int(x) for x in addr.split('.')] + assert len(octs) == 4, addr + for num in octs: + assert 0 <= num <= 255, addr + ipaddress.IPv4Address(addr) + elif family == socket.AF_INET6: + assert isinstance(addr, str), addr + ipaddress.IPv6Address(addr) + elif family == psutil.AF_LINK: + assert re.match(r'([a-fA-F0-9]{2}[:|\-]?){6}', addr) is not None, addr + else: + raise ValueError(f"unknown family {family!r}") + + +def check_connection_ntuple(conn): + """Check validity of a connection named tuple.""" + + def check_ntuple(conn): + has_pid = len(conn) == 7 + assert len(conn) in {6, 7}, len(conn) + assert conn[0] == conn.fd, conn.fd + assert conn[1] == conn.family, conn.family + assert conn[2] == conn.type, conn.type + assert conn[3] == conn.laddr, conn.laddr + assert conn[4] == conn.raddr, conn.raddr + assert conn[5] == conn.status, conn.status + if has_pid: + assert conn[6] == conn.pid, conn.pid + + def check_family(conn): + assert conn.family in {AF_INET, AF_INET6, AF_UNIX}, conn.family + assert isinstance(conn.family, enum.IntEnum), conn + if conn.family == AF_INET: + # actually try to bind the local socket; ignore IPv6 + # sockets as their address might be represented as + # an IPv4-mapped-address (e.g. "::127.0.0.1") + # and that's rejected by bind() + with socket.socket(conn.family, conn.type) as s: + try: + s.bind((conn.laddr[0], 0)) + except OSError as err: + if err.errno != errno.EADDRNOTAVAIL: + raise + elif conn.family == AF_UNIX: + assert conn.status == psutil.CONN_NONE, conn.status + + def check_type(conn): + # SOCK_SEQPACKET may happen in case of AF_UNIX socks + SOCK_SEQPACKET = getattr(socket, "SOCK_SEQPACKET", object()) + assert conn.type in { + socket.SOCK_STREAM, + socket.SOCK_DGRAM, + SOCK_SEQPACKET, + }, conn.type + assert isinstance(conn.type, enum.IntEnum), conn + if conn.type == socket.SOCK_DGRAM: + assert conn.status == psutil.CONN_NONE, conn.status + + def check_addrs(conn): + # check IP address and port sanity + for addr in (conn.laddr, conn.raddr): + if conn.family in {AF_INET, AF_INET6}: + assert isinstance(addr, tuple), type(addr) + if not addr: + continue + assert isinstance(addr.port, int), type(addr.port) + assert 0 <= addr.port <= 65535, addr.port + check_net_address(addr.ip, conn.family) + elif conn.family == AF_UNIX: + assert isinstance(addr, str), type(addr) + + def check_status(conn): + assert isinstance(conn.status, str), conn.status + valids = [ + getattr(psutil, x) for x in dir(psutil) if x.startswith('CONN_') + ] + assert conn.status in valids, conn.status + if conn.family in {AF_INET, AF_INET6} and conn.type == SOCK_STREAM: + assert conn.status != psutil.CONN_NONE, conn.status + else: + assert conn.status == psutil.CONN_NONE, conn.status + + check_ntuple_type_hints(conn) + check_ntuple(conn) + check_family(conn) + check_type(conn) + check_addrs(conn) + check_status(conn) + + +def filter_proc_net_connections(cons): + """Our process may start with some open UNIX sockets which are not + initialized by us, invalidating unit tests. + """ + new = [] + for conn in cons: + if POSIX and conn.family == socket.AF_UNIX: + if MACOS and "/syslog" in conn.raddr: + debug(f"skipping {conn}") + continue + new.append(conn) + return new + + +# ===================================================================== +# --- type hints +# ===================================================================== + + +class TypeHintsChecker: + try: + UNION_TYPES = (typing.Union, types.UnionType) + except AttributeError: # Python < 3.10 + UNION_TYPES = (typing.Union,) + + @staticmethod + @functools.lru_cache(maxsize=None) + def _get_ntuple_hints(nt): + cls = type(nt) + try: + localns = { + name: obj + for name, obj in vars(_enums).items() + if isinstance(obj, type) and issubclass(obj, enum.Enum) + } + localns['socket'] = socket + return typing.get_type_hints( + cls, + globalns=vars(ntuples), + localns=localns, + ) + except TypeError: + # Python < 3.10 can't evaluate "X | Y" union syntax. + return {} + + @staticmethod + def _hint_to_types(hint): + """Flatten a type hint into a tuple of concrete types suitable + for isinstance(). Returns None if the hint cannot be checked. + """ + origin = typing.get_origin(hint) + if origin in TypeHintsChecker.UNION_TYPES: + result = [] + for arg in typing.get_args(hint): + inner = typing.get_origin(arg) + if inner is not None: + result.append(inner) + elif isinstance(arg, type): + result.append(arg) + return tuple(result) if result else None + if origin is not None: + return (origin,) + if isinstance(hint, type): + return (hint,) + return None + + @staticmethod + def check_ntuple_type_hints(nt): + """Uses type hints from _ntuples.py to verify field types. `nt` + is a named tuple returned by one of psutil APIs. + """ + assert is_namedtuple(nt) + hints = TypeHintsChecker._get_ntuple_hints(nt) + if not hints: + return + for field in nt._fields: + if field not in hints: + # field is not annotated + continue + value = getattr(nt, field) + types_ = TypeHintsChecker._hint_to_types(hints[field]) + if types_ is None: + continue + # For IntEnum hints (e.g. socket.AddressFamily), psutil may + # return a platform-specific IntEnum subclass rather than + # the annotated one, so we broaden the check to int. + types_ = tuple( + ( + int + if isinstance(t, type) and issubclass(t, enum.IntEnum) + else t + ) + for t in types_ + ) + assert isinstance(value, types_), (field, value, types_) + + @staticmethod + @functools.lru_cache(maxsize=None) + def _get_return_hint(fun): + """Get the 'return' type hint for a psutil API function or + method. Resolves annotation strings using a combined namespace + of psutil globals (Any, Generator, Process, ...) and ntuple + types (scputimes, svmem, pmem, ...). Returns None if hints + cannot be resolved or there is no return annotation. + """ + while hasattr(fun, 'func'): + fun = fun.func + # Build a namespace that can resolve all annotations. + psp = vars(psutil).get('_psplatform') + psp_ns = vars(psp) if psp is not None else {} + ns = { + **psp_ns, + **vars(psutil), + **vars(ntuples), + **vars(typing), + } + underlying = getattr(fun, '__func__', fun) + try: + hints = typing.get_type_hints(underlying, globalns=ns) + except TypeError: + # X | Y union syntax in annotations requires Python 3.10+ + # to evaluate. On older versions skip the check entirely. + if sys.version_info < (3, 10): + msg = f"skip X|Y type check on old python for {fun.__name__!r}" + warn(msg) + return None + else: + raise + return hints.get('return') + + @staticmethod + def _check_container_items(hint, value): + """For list[T] and dict[K, V] hints, verify element types.""" + origin = typing.get_origin(hint) + args = typing.get_args(hint) + if origin is list and args: + elem_types = TypeHintsChecker._hint_to_types(args[0]) + if elem_types: + for item in value: + assert isinstance(item, elem_types), (item, elem_types) + elif origin is dict and len(args) == 2: + key_types = TypeHintsChecker._hint_to_types(args[0]) + val_types = TypeHintsChecker._hint_to_types(args[1]) + for k, v in value.items(): + if key_types: + assert isinstance(k, key_types), (k, key_types) + if val_types: + assert isinstance(v, val_types), (v, val_types) + + @staticmethod + def check_fun_type_hints(fun, retval): + """Use the 'return' type hint of *fun* from psutil/__init__.py + to verify that *retval* is an instance of the annotated type. + """ + hint = TypeHintsChecker._get_return_hint(fun) + if hint is None: + if not hasattr(types, "UnionType"): + # added in python 3.10 + return + raise ValueError(f"no type hints defined for {fun}") + types_ = TypeHintsChecker._hint_to_types(hint) + assert types_, hint + assert isinstance(retval, types_), (fun, retval, types_) + TypeHintsChecker._check_container_items(hint, retval) + + +check_ntuple_type_hints = TypeHintsChecker.check_ntuple_type_hints +check_fun_type_hints = TypeHintsChecker.check_fun_type_hints + + +# =================================================================== +# --- import utils +# =================================================================== + + +def reload_module(module): + return importlib.reload(module) + + +def import_module_by_path(path): + name = os.path.splitext(os.path.basename(path))[0] + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +# =================================================================== +# --- others +# =================================================================== + + +def warn(msg): + """Raise a warning msg.""" + warnings.warn(msg, UserWarning, stacklevel=2) + + +def is_namedtuple(x): + """Check if object is an instance of named tuple.""" + t = type(x) + if tuple not in t.__mro__: + return False + f = getattr(t, '_fields', None) + if not isinstance(f, tuple): + return False + return all(isinstance(n, str) for n in f) + + +if POSIX: + + @contextlib.contextmanager + def copyload_shared_lib(suffix=""): + """Ctx manager which picks up a random shared CO lib used + by this process, copies it in another location and loads it + in memory via ctypes. Return the new absolutized path. + """ + exe = 'pypy' if PYPY else 'python' + ext = ".so" + dst = get_testfn(suffix=suffix + ext) + libs = [ + x.path + for x in psutil.Process().memory_maps() + if os.path.splitext(x.path)[1] == ext and exe in x.path.lower() + ] + src = random.choice(libs) + shutil.copyfile(src, dst) + try: + ctypes.CDLL(dst) + yield dst + finally: + safe_rmpath(dst) + +else: + + @contextlib.contextmanager + def copyload_shared_lib(suffix=""): + """Ctx manager which picks up a random shared DLL lib used + by this process, copies it in another location and loads it + in memory via ctypes. + Return the new absolutized, normcased path. + """ + from ctypes import WinError + from ctypes import wintypes + + ext = ".dll" + dst = get_testfn(suffix=suffix + ext) + libs = [ + x.path + for x in psutil.Process().memory_maps() + if x.path.lower().endswith(ext) + and 'python' in os.path.basename(x.path).lower() + ] + if PYPY and not libs: + libs = [ + x.path + for x in psutil.Process().memory_maps() + if 'pypy' in os.path.basename(x.path).lower() + ] + src = random.choice(libs) + shutil.copyfile(src, dst) + cfile = None + try: + cfile = ctypes.WinDLL(dst) + yield dst + finally: + # Work around OverflowError: + # - https://ci.appveyor.com/project/giampaolo/psutil/build/1207/ + # job/o53330pbnri9bcw7 + # - http://bugs.python.org/issue30286 + # - http://stackoverflow.com/questions/23522055 + if cfile is not None: + FreeLibrary = ctypes.windll.kernel32.FreeLibrary + FreeLibrary.argtypes = [wintypes.HMODULE] + ret = FreeLibrary(cfile._handle) + if ret == 0: + raise WinError() + safe_rmpath(dst) + + +# =================================================================== +# --- Exit funs (first is executed last) +# =================================================================== + + +# this is executed first +@atexit.register +def cleanup_test_procs(): + reap_children(recursive=True) + + +# atexit module does not execute exit functions in case of SIGTERM, which +# gets sent to test subprocesses, which is a problem if they import this +# module. With this it will. See: +# https://gmpy.dev/blog/2016/how-to-always-execute-exit-functions-in-python +if POSIX: + signal.signal(signal.SIGTERM, lambda sig, _: sys.exit(sig)) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..e19aaa7303 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,65 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Pytest hooks.""" + +import warnings + +import pytest + +# Activate pytest hooks defined in test_process_all.py. +from .test_process_all import pytest_runtest_makereport # noqa: F401 +from .test_process_all import pytest_terminal_summary # noqa: F401 + + +def _escape_surrogates(text): + try: + text.encode("utf-8") + except UnicodeEncodeError: + return text.encode("utf-8", "backslashreplace").decode("utf-8") + else: + return text + + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_logreport(report): + """Escape the non-UTF8 paths used by test_unicode.py when a test + fails. If one of them ends up in a report, xdist can't send it to + the master process, and the whole run dies. + """ + if isinstance(report.longrepr, tuple): + path, lineno, reason = report.longrepr + report.longrepr = (path, lineno, _escape_surrogates(reason)) + elif report.longrepr is not None: + text = str(report.longrepr) + escaped = _escape_surrogates(text) + if escaped != text: + report.longrepr = escaped + report.sections = [ + (name, _escape_surrogates(content)) + for name, content in report.sections + ] + + +# Monkey patch pytest-instafail so that we ALSO get the full +# traceback/failure summary at the end of the run, see: +# https://github.com/pytest-dev/pytest-instafail/issues/21. +try: + import pytest_instafail + from _pytest.terminal import TerminalReporter + + pytest_instafail.InstafailingTerminalReporter # noqa: B018 +except (ImportError, AttributeError): + warnings.warn( + "failed to monkey patch pytest-instafail", + category=DeprecationWarning, + stacklevel=2, + ) +else: + pytest_instafail.InstafailingTerminalReporter.summary_failures = ( + TerminalReporter.summary_failures + ) + pytest_instafail.InstafailingTerminalReporter.summary_errors = ( + TerminalReporter.summary_errors + ) diff --git a/tests/test_aix.py b/tests/test_aix.py new file mode 100755 index 0000000000..6d9c72f585 --- /dev/null +++ b/tests/test_aix.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola' +# Copyright (c) 2017, Arnon Yaari +# All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""AIX specific tests.""" + +import re + +import psutil +from psutil import AIX + +from . import PsutilTestCase +from . import sh +from . import skipif + + +@skipif(not AIX, reason="AIX only") +class AIXSpecificTestCase(PsutilTestCase): + def test_virtual_memory(self): + out = sh('/usr/bin/svmon -O unit=KB') + re_pattern = r"memory\s*" + for field in [ + "size", + "inuse", + "free", + "pin", + "virtual", + "available", + "mmode", + ]: + re_pattern += rf"(?P<{field}>\S+)\s+" + matchobj = re.search(re_pattern, out) + + assert matchobj is not None + + KB = 1024 + total = int(matchobj.group("size")) * KB + available = int(matchobj.group("available")) * KB + used = int(matchobj.group("inuse")) * KB + free = int(matchobj.group("free")) * KB + + psutil_result = psutil.virtual_memory() + + # TOLERANCE_SYS_MEM is not enough. For some reason we're seeing + # differences of ~1.2 MB. 2 MB is still a good tolerance when + # compared to GBs. + TOLERANCE_SYS_MEM = 2 * KB * KB # 2 MB + assert psutil_result.total == total + assert abs(psutil_result.used - used) < TOLERANCE_SYS_MEM + assert abs(psutil_result.available - available) < TOLERANCE_SYS_MEM + assert abs(psutil_result.free - free) < TOLERANCE_SYS_MEM + + def test_swap_memory(self): + out = sh('/usr/sbin/lsps -a') + # From the man page, "The size is given in megabytes" so we assume + # we'll always have 'MB' in the result + # TODO maybe try to use "swap -l" to check "used" too, but its units + # are not guaranteed to be "MB" so parsing may not be consistent + matchobj = re.search( + r"(?P\S+)\s+" + r"(?P\S+)\s+" + r"(?P\S+)\s+" + r"(?P\d+)MB", + out, + ) + + assert matchobj is not None + + total_mb = int(matchobj.group("size")) + MB = 1024**2 + psutil_result = psutil.swap_memory() + # we divide our result by MB instead of multiplying the lsps value by + # MB because lsps may round down, so we round down too + assert int(psutil_result.total / MB) == total_mb + + def test_cpu_stats(self): + out = sh('/usr/bin/mpstat -a') + + re_pattern = r"ALL\s*" + for field in [ + "min", + "maj", + "mpcs", + "mpcr", + "dev", + "soft", + "dec", + "ph", + "cs", + "ics", + "bound", + "rq", + "push", + "S3pull", + "S3grd", + "S0rd", + "S1rd", + "S2rd", + "S3rd", + "S4rd", + "S5rd", + "sysc", + ]: + re_pattern += rf"(?P<{field}>\S+)\s+" + matchobj = re.search(re_pattern, out) + + assert matchobj is not None + + # numbers are usually in the millions so 1000 is ok for tolerance + CPU_STATS_TOLERANCE = 1000 + psutil_result = psutil.cpu_stats() + assert ( + abs(psutil_result.ctx_switches - int(matchobj.group("cs"))) + < CPU_STATS_TOLERANCE + ) + assert ( + abs(psutil_result.syscalls - int(matchobj.group("sysc"))) + < CPU_STATS_TOLERANCE + ) + assert ( + abs(psutil_result.interrupts - int(matchobj.group("dev"))) + < CPU_STATS_TOLERANCE + ) + assert ( + abs(psutil_result.soft_interrupts - int(matchobj.group("soft"))) + < CPU_STATS_TOLERANCE + ) + + def test_cpu_count_logical(self): + out = sh('/usr/bin/mpstat -a') + mpstat_lcpu = int(re.search(r"lcpu=(\d+)", out).group(1)) + psutil_lcpu = psutil.cpu_count(logical=True) + assert mpstat_lcpu == psutil_lcpu + + def test_net_if_addrs_names(self): + out = sh('/etc/ifconfig -l') + ifconfig_names = set(out.split()) + psutil_names = set(psutil.net_if_addrs().keys()) + assert ifconfig_names == psutil_names diff --git a/tests/test_bsd.py b/tests/test_bsd.py new file mode 100755 index 0000000000..a83057ca9e --- /dev/null +++ b/tests/test_bsd.py @@ -0,0 +1,656 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# TODO: (FreeBSD) add test for comparing connections with 'sockstat' cmd. + + +"""Tests specific to all BSD platforms.""" + +import datetime +import os +import re +import time +from unittest import mock + +import psutil +from psutil import BSD +from psutil import FREEBSD +from psutil import NETBSD +from psutil import OPENBSD +from psutil import _psutil + +from . import HAS_BATTERY +from . import TOLERANCE_SYS_MEM +from . import PsutilTestCase +from . import isolated +from . import pytest +from . import requires_cli +from . import retry_on_failure +from . import sh +from . import skipif +from . import spawn_subproc +from . import terminate + +PAGESIZE = _psutil.getpagesize() if BSD else None + + +def sysctl(cmdline): + """Expects a sysctl command with an argument and parse the result + returning only the value of interest. + """ + result = sh("sysctl " + cmdline) + if FREEBSD: + result = result[result.find(": ") + 2 :] + elif OPENBSD or NETBSD: + result = result[result.find("=") + 1 :] + try: + return int(result) + except ValueError: + return result + + +# ===================================================================== +# --- All BSD* +# ===================================================================== + + +@skipif(not BSD, reason="BSD only") +class TestSystemAPIs(PsutilTestCase): + """System tests common to all BSD variants.""" + + def test_disks(self): + # test psutil.disk_usage() and psutil.disk_partitions() + # against "df -a" + def df(path): + out = sh(f'df -k "{path}"').strip() + lines = out.split('\n') + lines.pop(0) + line = lines.pop(0) + dev, total, used, free = line.split()[:4] + if dev == 'none': + dev = '' + total = int(total) * 1024 + used = int(used) * 1024 + free = int(free) * 1024 + return dev, total, used, free + + for part in psutil.disk_partitions(all=False): + usage = psutil.disk_usage(part.mountpoint) + dev, total, used, free = df(part.mountpoint) + assert part.device == dev + assert usage.total == total + # 10 MB tolerance + if abs(usage.free - free) > 10 * 1024 * 1024: + return pytest.fail(f"psutil={usage.free}, df={free}") + if abs(usage.used - used) > 10 * 1024 * 1024: + return pytest.fail(f"psutil={usage.used}, df={used}") + + @requires_cli("sysctl") + def test_cpu_count_logical(self): + syst = sysctl("hw.ncpu") + assert psutil.cpu_count(logical=True) == syst + + # On NetBSD total is UVM's managed pages, which is less than + # physical RAM. NetBSDTestCase.test_vmem_total covers it instead. + @requires_cli("sysctl") + @skipif(NETBSD, reason="hw.physmem is not what psutil reports") + def test_virtual_memory_total(self): + num = sysctl('hw.physmem') + assert num == psutil.virtual_memory().total + + @requires_cli("ifconfig") + def test_net_if_stats(self): + for name, stats in psutil.net_if_stats().items(): + try: + out = sh(f"ifconfig {name}") + except RuntimeError: + pass + else: + assert stats.isup == ('RUNNING' in out) + if "mtu" in out: + assert stats.mtu == int(re.findall(r'mtu (\d+)', out)[0]) + + +@skipif(not BSD, reason="BSD only") +class TestProcessAPIs(PsutilTestCase): + + @classmethod + def setUpClass(cls): + cls.pid = spawn_subproc().pid + + @classmethod + def tearDownClass(cls): + terminate(cls.pid) + + @skipif(NETBSD, reason="-o lstart doesn't work on NETBSD") + def test_create_time(self): + output = sh(f"ps -o lstart -p {self.pid}") + start_ps = output.replace('STARTED', '').strip() + start_psutil = psutil.Process(self.pid).create_time() + start_psutil = time.strftime( + "%a %b %e %H:%M:%S %Y", time.localtime(start_psutil) + ) + assert start_ps == start_psutil + + def test_environ_zombie(self): + _parent, zombie = self.spawn_zombie() + with pytest.raises(psutil.ZombieProcess): + zombie.environ() + + +@skipif(not BSD, reason="BSD only") +class TestVmstat(PsutilTestCase): + + @staticmethod + def vmstat(labels): + out = sh(["vmstat", "-s"], env={"LANG": "C.UTF-8"}) + for line in out.split("\n"): + line = line.strip() + num, _, what = line.partition(" ") + for label in labels: + if label == what: + return int(num) + return pytest.skip(f"can't find {labels} in vmstat output") + + # --- virtual_memory() + + def test_vmem_free(self): + vmstat_value = self.vmstat(['pages free']) * PAGESIZE + psutil_value = psutil.virtual_memory().free + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + def test_vmem_active(self): + vmstat_value = self.vmstat(['pages active']) * PAGESIZE + psutil_value = psutil.virtual_memory().active + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + def test_vmem_inactive(self): + vmstat_value = self.vmstat(['pages inactive']) * PAGESIZE + psutil_value = psutil.virtual_memory().inactive + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + def test_vmem_cached(self): + # NetBSD / OpenBSD + vmstat_value = ( + self.vmstat(['cached file pages']) + + self.vmstat(['cached executable pages']) + ) * PAGESIZE + psutil_value = psutil.virtual_memory().cached + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + def test_vmem_wired(self): + vmstat_value = ( + self.vmstat(['pages wired', 'pages wired down']) * PAGESIZE + ) + psutil_value = psutil.virtual_memory().wired + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + @skipif(not (OPENBSD or NETBSD), reason="NETBSD / OPENBSD only") + def test_vmem_shared(self): + out = sh("vmstat -t") + if "vm-sh" not in out: + return pytest.skip("can't find 'vm-sh' in vmstat output") + lines = out.splitlines() + headers = lines[1].split() + values = lines[2].split() + row = dict(zip(headers, values)) + expected = int(row["vm-sh"]) * PAGESIZE + assert ( + abs(psutil.virtual_memory().shared - expected) < TOLERANCE_SYS_MEM + ) + + # --- swap_memory() + + def test_swap_total(self): + vmstat_value = self.vmstat(['swap pages']) * PAGESIZE + psutil_value = psutil.swap_memory().total + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + def test_swap_used(self): + vmstat_value = self.vmstat(['swap pages in use']) * PAGESIZE + psutil_value = psutil.swap_memory().used + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + def test_swap_sin(self): + vmstat_value = self.vmstat(['pages swapped in']) * PAGESIZE + psutil_value = psutil.swap_memory().sin + assert abs(vmstat_value - psutil_value) < 1024 + + def test_swap_sout(self): + vmstat_value = self.vmstat(['pages swapped oud']) * PAGESIZE + psutil_value = psutil.swap_memory().sout + assert abs(vmstat_value - psutil_value) < 1024 + + # --- cpu_stats() + + @isolated + @retry_on_failure + def test_cpu_stats_interrupts(self): + vmstat_value = self.vmstat(['device interrupts', 'interrupts']) + psutil_value = psutil.cpu_stats().interrupts + assert abs(vmstat_value - psutil_value) <= 100 + + @isolated + @retry_on_failure + def test_cpu_stats_soft_interrupts(self): + vmstat_value = self.vmstat(['software interrupts']) + psutil_value = psutil.cpu_stats().soft_interrupts + assert abs(vmstat_value - psutil_value) <= 100 + + @isolated + @retry_on_failure + def test_cpu_stats_syscalls(self): + vmstat_value = self.vmstat(['system calls', 'syscalls']) + psutil_value = psutil.cpu_stats().syscalls + assert abs(vmstat_value - psutil_value) <= 100 + + @isolated + @retry_on_failure + def test_cpu_stats_ctx_switches(self): + vmstat_value = self.vmstat(['cpu context switches']) + psutil_value = psutil.cpu_stats().ctx_switches + assert abs(vmstat_value - psutil_value) <= 100 + + +# ===================================================================== +# --- FreeBSD +# ===================================================================== + + +@skipif(not FREEBSD, reason="FREEBSD only") +class FreeBSDProcessTestCase(PsutilTestCase): + @classmethod + def setUpClass(cls): + cls.pid = spawn_subproc().pid + + @classmethod + def tearDownClass(cls): + terminate(cls.pid) + + @retry_on_failure + def test_memory_maps(self): + out = sh(f"procstat -v {self.pid}") + maps = psutil.Process(self.pid).memory_maps(grouped=False) + lines = out.split('\n')[1:] + while lines: + line = lines.pop() + fields = line.split() + _, start, stop, _perms, res = fields[:5] + map = maps.pop() + assert f"{start}-{stop}" == map.addr + assert int(res) * PAGESIZE == map.rss + if not map.path.startswith('['): + assert fields[10] == map.path + + def test_exe(self): + out = sh(f"procstat -b {self.pid}") + assert psutil.Process(self.pid).exe() == out.split('\n')[1].split()[-1] + + def test_cmdline(self): + out = sh(f"procstat -c {self.pid}") + assert ' '.join(psutil.Process(self.pid).cmdline()) == ' '.join( + out.split('\n')[1].split()[2:] + ) + + def test_uids_gids(self): + out = sh(f"procstat -s {self.pid}") + euid, ruid, suid, egid, rgid, sgid = out.split('\n')[1].split()[2:8] + p = psutil.Process(self.pid) + uids = p.uids() + gids = p.gids() + assert uids.real == int(ruid) + assert uids.effective == int(euid) + assert uids.saved == int(suid) + assert gids.real == int(rgid) + assert gids.effective == int(egid) + assert gids.saved == int(sgid) + + @retry_on_failure + def test_ctx_switches(self): + tested = [] + out = sh(f"procstat -r {self.pid}") + p = psutil.Process(self.pid) + for line in out.split('\n'): + line = line.lower().strip() + if ' voluntary context' in line: + pstat_value = int(line.split()[-1]) + psutil_value = p.num_ctx_switches().voluntary + assert pstat_value == psutil_value + tested.append(None) + elif ' involuntary context' in line: + pstat_value = int(line.split()[-1]) + psutil_value = p.num_ctx_switches().involuntary + assert pstat_value == psutil_value + tested.append(None) + if len(tested) != 2: + raise RuntimeError("couldn't find lines match in procstat out") + + @retry_on_failure + def test_cpu_times(self): + tested = [] + out = sh(f"procstat -r {self.pid}") + p = psutil.Process(self.pid) + for line in out.split('\n'): + line = line.lower().strip() + if 'user time' in line: + pstat_value = float('0.' + line.split()[-1].split('.')[-1]) + psutil_value = p.cpu_times().user + assert pstat_value == psutil_value + tested.append(None) + elif 'system time' in line: + pstat_value = float('0.' + line.split()[-1].split('.')[-1]) + psutil_value = p.cpu_times().system + assert pstat_value == psutil_value + tested.append(None) + if len(tested) != 2: + raise RuntimeError("couldn't find lines match in procstat out") + + +@skipif(not FREEBSD, reason="FREEBSD only") +class FreeBSDSystemTestCase(PsutilTestCase): + @staticmethod + def parse_swapinfo(): + # the last line is always the total + output = sh("swapinfo -k").splitlines()[-1] + parts = re.split(r'\s+', output) + + if not parts: + raise ValueError(f"Can't parse swapinfo: {output}") + + # the size is in 1k units, so multiply by 1024 + total, used, free = (int(p) * 1024 for p in parts[1:4]) + return total, used, free + + def test_cpu_count_cores(self): + cores = sysctl("kern.smp.cores") + assert psutil.cpu_count(logical=False) == cores + + @retry_on_failure + def test_cpu_times(self): + clk_tck = os.sysconf("SC_CLK_TCK") + ticks = [int(x) for x in sysctl("kern.cp_time").split()] + ct = psutil.cpu_times() + tolerance = 0.5 + assert abs(ct.user - ticks[0] / clk_tck) < tolerance + assert abs(ct.nice - ticks[1] / clk_tck) < tolerance + assert abs(ct.system - ticks[2] / clk_tck) < tolerance + assert abs(ct.irq - ticks[3] / clk_tck) < tolerance + assert abs(ct.idle - ticks[4] / clk_tck) < tolerance + + def test_cpu_frequency_against_sysctl(self): + # Currently only cpu 0 is frequency is supported in FreeBSD + # All other cores use the same frequency. + sensor = "dev.cpu.0.freq" + try: + sysctl_result = int(sysctl(sensor)) + except RuntimeError: + return pytest.skip("frequencies not supported by kernel") + assert psutil.cpu_freq().current == sysctl_result + + sensor = "dev.cpu.0.freq_levels" + sysctl_result = sysctl(sensor) + # sysctl returns a string of the format: + # / /... + # Ordered highest available to lowest available. + max_freq = int(sysctl_result.split()[0].split("/")[0]) + min_freq = int(sysctl_result.split()[-1].split("/")[0]) + assert psutil.cpu_freq().max == max_freq + assert psutil.cpu_freq().min == min_freq + + def test_cpu_freq_no_levels(self): + # No freq_levels means we can't tell min / max. It used to + # raise NameError, or reuse the previous CPU's values. + with mock.patch.object( + _psutil, "cpu_freq", return_value=(100, "") + ) as m: + ret = psutil._psbsd.cpu_freq() + assert m.called + for nt in ret: + assert nt.current == 100 + assert nt.min is None + assert nt.max is None + + # --- virtual_memory(); tests against sysctl + + @retry_on_failure + def test_vmem_active(self): + syst = sysctl("vm.stats.vm.v_active_count") * PAGESIZE + assert abs(psutil.virtual_memory().active - syst) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_vmem_inactive(self): + syst = sysctl("vm.stats.vm.v_inactive_count") * PAGESIZE + assert abs(psutil.virtual_memory().inactive - syst) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_vmem_wired(self): + syst = sysctl("vm.stats.vm.v_wire_count") * PAGESIZE + assert abs(psutil.virtual_memory().wired - syst) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_vmem_cached(self): + syst = sysctl("vm.stats.vm.v_cache_count") * PAGESIZE + assert abs(psutil.virtual_memory().cached - syst) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_vmem_free(self): + syst = sysctl("vm.stats.vm.v_free_count") * PAGESIZE + assert abs(psutil.virtual_memory().free - syst) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_vmem_buffers(self): + syst = sysctl("vfs.bufspace") + assert abs(psutil.virtual_memory().buffers - syst) < TOLERANCE_SYS_MEM + + def test_cpu_stats_ctx_switches(self): + assert ( + abs( + psutil.cpu_stats().ctx_switches + - sysctl('vm.stats.sys.v_swtch') + ) + < 1000 + ) + + def test_cpu_stats_interrupts(self): + assert ( + abs(psutil.cpu_stats().interrupts - sysctl('vm.stats.sys.v_intr')) + < 1000 + ) + + def test_cpu_stats_soft_interrupts(self): + assert ( + abs( + psutil.cpu_stats().soft_interrupts + - sysctl('vm.stats.sys.v_soft') + ) + < 1000 + ) + + @retry_on_failure + def test_cpu_stats_syscalls(self): + # pretty high tolerance but it looks like it's OK. + assert ( + abs(psutil.cpu_stats().syscalls - sysctl('vm.stats.sys.v_syscall')) + < 200000 + ) + + # --- swap memory + + def test_swapmem_free(self): + _total, _used, free = self.parse_swapinfo() + assert abs(psutil.swap_memory().free - free) < TOLERANCE_SYS_MEM + + def test_swapmem_used(self): + _total, used, _free = self.parse_swapinfo() + assert abs(psutil.swap_memory().used - used) < TOLERANCE_SYS_MEM + + def test_swapmem_total(self): + total, _used, _free = self.parse_swapinfo() + assert abs(psutil.swap_memory().total - total) < TOLERANCE_SYS_MEM + + # --- net + + @retry_on_failure + def test_net_io_counters(self): + out = sh("netstat -ib") + netstat = {} + for line in out.splitlines(): + fields = line.split() + if len(fields) == 12 and " ' + Mirrors the same uvmexp_sysctl fields that psutil reads via + sysctl(CTL_VM, VM_UVMEXP2) in C, without requiring procfs. + """ + out = sh("vmstat -s") + for line in out.splitlines(): + line = line.strip() + if look_for in line: + return int(line.split()[0]) + raise ValueError(f"can't find {look_for!r} in vmstat -s output") + + # --- virtual mem + + def test_vmem_total(self): + num = self.parse_vmstat("pages managed") + assert num * PAGESIZE == psutil.virtual_memory().total + + @retry_on_failure + def test_vmem_buffers(self): + # uv.filepages: file-backed pages excluding executable mappings + assert ( + abs( + psutil.virtual_memory().buffers + - self.parse_vmstat("cached file pages") * PAGESIZE + ) + < TOLERANCE_SYS_MEM + ) + + # --- swap mem + + @retry_on_failure + def test_swapmem_total(self): + assert ( + abs( + psutil.swap_memory().total + - self.parse_vmstat("swap pages") * PAGESIZE + ) + < TOLERANCE_SYS_MEM + ) diff --git a/tests/test_connections.py b/tests/test_connections.py new file mode 100755 index 0000000000..d4943ec983 --- /dev/null +++ b/tests/test_connections.py @@ -0,0 +1,582 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for psutil.net_connections() and Process.net_connections() APIs.""" + +import os +import socket +import subprocess +import textwrap +from contextlib import closing +from socket import AF_INET +from socket import AF_INET6 +from socket import SOCK_DGRAM +from socket import SOCK_STREAM + +import psutil +from psutil import FREEBSD +from psutil import LINUX +from psutil import MACOS +from psutil import NETBSD +from psutil import OPENBSD +from psutil import POSIX +from psutil import SUNOS +from psutil import WINDOWS +from psutil._common import supports_ipv6 + +from . import AF_UNIX +from . import HAS_NET_CONNECTIONS_UNIX +from . import ROOT_DIR +from . import SKIP_SYSCONS +from . import PsutilTestCase +from . import bind_socket +from . import bind_unix_socket +from . import check_connection_ntuple +from . import create_sockets +from . import filter_proc_net_connections +from . import pytest +from . import reap_children +from . import retry_on_failure +from . import serial +from . import skip_on_access_denied +from . import skipif +from . import tcp_socketpair +from . import unix_socketpair +from . import wait_for_file_subproc + +SOCK_SEQPACKET = getattr(socket, "SOCK_SEQPACKET", object()) + + +def this_proc_net_connections(kind): + cons = psutil.Process().net_connections(kind=kind) + if kind in {"all", "unix"}: + return filter_proc_net_connections(cons) + return cons + + +@serial +class ConnectionTestCase(PsutilTestCase): + def setUp(self): + assert this_proc_net_connections(kind='all') == [] + + def tearDown(self): + # Make sure we closed all resources. + assert this_proc_net_connections(kind='all') == [] + + def compare_procsys_connections(self, pid, proc_cons, kind='all'): + """Given a process PID and its list of connections compare + those against system-wide connections retrieved via + psutil.net_connections. + """ + try: + sys_cons = psutil.net_connections(kind=kind) + except psutil.AccessDenied: + # On MACOS, system-wide connections are retrieved by iterating + # over all processes + if MACOS: + return + else: + raise + # Filter for this proc PID and exclude PIDs from the tuple. + sys_cons = [c[:-1] for c in sys_cons if c.pid == pid] + sys_cons.sort() + proc_cons.sort() + assert proc_cons == sys_cons + + +class TestBasicOperations(ConnectionTestCase): + @skipif(SKIP_SYSCONS, reason="requires root") + def test_system(self): + with create_sockets(): + for conn in psutil.net_connections(kind='all'): + check_connection_ntuple(conn) + + def test_process(self): + with create_sockets(): + for conn in this_proc_net_connections(kind='all'): + check_connection_ntuple(conn) + + def test_invalid_kind(self): + with pytest.raises(ValueError): + this_proc_net_connections(kind='???') + with pytest.raises(ValueError): + psutil.net_connections(kind='???') + + +@serial +class TestUnconnectedSockets(ConnectionTestCase): + """Tests sockets which are open but not connected to anything.""" + + def get_conn_from_sock(self, sock): + cons = this_proc_net_connections(kind='all') + smap = {c.fd: c for c in cons} + if NETBSD or FREEBSD: + # NetBSD opens a UNIX socket to /var/log/run + # so there may be more connections. + return smap[sock.fileno()] + else: + assert len(cons) == 1 + if cons[0].fd != -1: + assert smap[sock.fileno()].fd == sock.fileno() + return cons[0] + + def check_socket(self, sock): + """Given a socket, makes sure it matches the one obtained + via psutil. It assumes this process created one connection + only (the one supposed to be checked). + """ + conn = self.get_conn_from_sock(sock) + check_connection_ntuple(conn) + + # fd, family, type + if conn.fd != -1: + assert conn.fd == sock.fileno() + assert conn.family == sock.family + # see: http://bugs.python.org/issue30204 + assert conn.type == sock.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE) + + # local address + laddr = sock.getsockname() + if not laddr and isinstance(laddr, bytes): + # See: http://bugs.python.org/issue30205 + laddr = laddr.decode() + if sock.family == AF_INET6: + laddr = laddr[:2] + assert conn.laddr == laddr + + # XXX Solaris can't retrieve system-wide UNIX sockets + if sock.family == AF_UNIX and HAS_NET_CONNECTIONS_UNIX: + cons = this_proc_net_connections(kind='all') + self.compare_procsys_connections(os.getpid(), cons, kind='all') + return conn + + def test_tcp_v4(self): + addr = ("127.0.0.1", 0) + with closing(bind_socket(AF_INET, SOCK_STREAM, addr=addr)) as sock: + conn = self.check_socket(sock) + assert conn.raddr == () + assert conn.status == psutil.CONN_LISTEN + + @skipif(not supports_ipv6(), reason="IPv6 not supported") + def test_tcp_v6(self): + addr = ("::1", 0) + with closing(bind_socket(AF_INET6, SOCK_STREAM, addr=addr)) as sock: + conn = self.check_socket(sock) + assert conn.raddr == () + assert conn.status == psutil.CONN_LISTEN + + def test_udp_v4(self): + addr = ("127.0.0.1", 0) + with closing(bind_socket(AF_INET, SOCK_DGRAM, addr=addr)) as sock: + conn = self.check_socket(sock) + assert conn.raddr == () + assert conn.status == psutil.CONN_NONE + + @skipif(not supports_ipv6(), reason="IPv6 not supported") + def test_udp_v6(self): + addr = ("::1", 0) + with closing(bind_socket(AF_INET6, SOCK_DGRAM, addr=addr)) as sock: + conn = self.check_socket(sock) + assert conn.raddr == () + assert conn.status == psutil.CONN_NONE + + @skipif(not POSIX, reason="POSIX only") + def test_unix_tcp(self): + testfn = self.get_testfn() + with closing(bind_unix_socket(testfn, type=SOCK_STREAM)) as sock: + conn = self.check_socket(sock) + assert conn.raddr == "" + assert conn.status == psutil.CONN_NONE + + @skipif(not POSIX, reason="POSIX only") + def test_unix_udp(self): + testfn = self.get_testfn() + with closing(bind_unix_socket(testfn, type=SOCK_STREAM)) as sock: + conn = self.check_socket(sock) + assert conn.raddr == "" + assert conn.status == psutil.CONN_NONE + + +@serial +class TestConnectedSocket(ConnectionTestCase): + """Test socket pairs which are actually connected to + each other. + """ + + # On SunOS, even after we close() it, the server socket stays around + # in TIME_WAIT state. + @skipif(SUNOS, reason="unreliable on SUNOS") + def test_tcp(self): + addr = ("127.0.0.1", 0) + assert this_proc_net_connections(kind='tcp4') == [] + server, client = tcp_socketpair(AF_INET, addr=addr) + try: + cons = this_proc_net_connections(kind='tcp4') + assert len(cons) == 2 + assert cons[0].status == psutil.CONN_ESTABLISHED + assert cons[1].status == psutil.CONN_ESTABLISHED + # May not be fast enough to change state so it stays + # commented. + # client.close() + # cons = this_proc_net_connections(kind='all') + # assert len(cons) == 1 + # assert cons[0].status == psutil.CONN_CLOSE_WAIT + finally: + server.close() + client.close() + + @skipif(not POSIX, reason="POSIX only") + @skipif(not HAS_NET_CONNECTIONS_UNIX, reason="can't list UNIX sockets") + def test_unix(self): + testfn = self.get_testfn() + server, client = unix_socketpair(testfn) + try: + cons = this_proc_net_connections(kind='unix') + assert not (cons[0].laddr and cons[0].raddr), cons + assert not (cons[1].laddr and cons[1].raddr), cons + if NETBSD or FREEBSD: + # On NetBSD creating a UNIX socket will cause + # a UNIX connection to /var/run/log. + cons = [c for c in cons if c.raddr != '/var/run/log'] + assert len(cons) == 2 + if LINUX or FREEBSD or SUNOS or OPENBSD: + # remote path is never set + assert cons[0].raddr == "" + assert cons[1].raddr == "" + # one local address should though + assert testfn == (cons[0].laddr or cons[1].laddr) + else: + # On other systems either the laddr or raddr + # of both peers are set. + assert (cons[0].laddr or cons[1].laddr) == testfn + finally: + server.close() + client.close() + + +class TestFilters(ConnectionTestCase): + def test_filters(self): + def check(kind, families, types): + for conn in this_proc_net_connections(kind=kind): + assert conn.family in families + assert conn.type in types + if not SKIP_SYSCONS: + for conn in psutil.net_connections(kind=kind): + assert conn.family in families + assert conn.type in types + + with create_sockets(): + check( + 'all', + [AF_INET, AF_INET6, AF_UNIX], + [SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET], + ) + check('inet', [AF_INET, AF_INET6], [SOCK_STREAM, SOCK_DGRAM]) + check('inet4', [AF_INET], [SOCK_STREAM, SOCK_DGRAM]) + check('tcp', [AF_INET, AF_INET6], [SOCK_STREAM]) + check('tcp4', [AF_INET], [SOCK_STREAM]) + check('tcp6', [AF_INET6], [SOCK_STREAM]) + check('udp', [AF_INET, AF_INET6], [SOCK_DGRAM]) + check('udp4', [AF_INET], [SOCK_DGRAM]) + check('udp6', [AF_INET6], [SOCK_DGRAM]) + if HAS_NET_CONNECTIONS_UNIX: + check( + 'unix', + [AF_UNIX], + [SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET], + ) + + @skip_on_access_denied(only_if=MACOS) + def test_combos(self): + reap_children() + + def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): + all_kinds = ( + "all", + "inet", + "inet4", + "inet6", + "tcp", + "tcp4", + "tcp6", + "udp", + "udp4", + "udp6", + ) + check_connection_ntuple(conn) + assert conn.family == family + assert conn.type == type + assert conn.laddr == laddr + assert conn.raddr == raddr + assert conn.status == status + for kind in all_kinds: + cons = proc.net_connections(kind=kind) + if kind in kinds: + assert cons != [] + else: + assert cons == [] + # compare against system-wide connections + # XXX Solaris can't retrieve system-wide UNIX + # sockets. + if HAS_NET_CONNECTIONS_UNIX: + self.compare_procsys_connections(proc.pid, [conn]) + + tcp_template = textwrap.dedent(""" + import socket, time + s = socket.socket({family}, socket.SOCK_STREAM) + s.bind(('{addr}', 0)) + s.listen(5) + with open('{testfn}', 'w') as f: + f.write(str(s.getsockname()[:2])) + [time.sleep(0.1) for x in range(100)] + """) + + udp_template = textwrap.dedent(""" + import socket, time + s = socket.socket({family}, socket.SOCK_DGRAM) + s.bind(('{addr}', 0)) + with open('{testfn}', 'w') as f: + f.write(str(s.getsockname()[:2])) + [time.sleep(0.1) for x in range(100)] + """) + + # must be relative on Windows + testfile = os.path.basename(self.get_testfn(dir=os.getcwd())) + tcp4_template = tcp_template.format( + family=int(AF_INET), addr="127.0.0.1", testfn=testfile + ) + udp4_template = udp_template.format( + family=int(AF_INET), addr="127.0.0.1", testfn=testfile + ) + tcp6_template = tcp_template.format( + family=int(AF_INET6), addr="::1", testfn=testfile + ) + udp6_template = udp_template.format( + family=int(AF_INET6), addr="::1", testfn=testfile + ) + + # launch various subprocess instantiating a socket of various + # families and types to enrich psutil results + tcp4_proc = self.pyrun(tcp4_template, stderr=subprocess.PIPE) + tcp4_addr = eval( + wait_for_file_subproc(testfile, tcp4_proc, delete=True) + ) + udp4_proc = self.pyrun(udp4_template, stderr=subprocess.PIPE) + udp4_addr = eval( + wait_for_file_subproc(testfile, udp4_proc, delete=True) + ) + if supports_ipv6(): + tcp6_proc = self.pyrun(tcp6_template, stderr=subprocess.PIPE) + tcp6_addr = eval( + wait_for_file_subproc(testfile, tcp6_proc, delete=True) + ) + udp6_proc = self.pyrun(udp6_template, stderr=subprocess.PIPE) + udp6_addr = eval( + wait_for_file_subproc(testfile, udp6_proc, delete=True) + ) + else: + tcp6_proc = None + udp6_proc = None + tcp6_addr = None + udp6_addr = None + + for p in psutil.Process().children(): + cons = p.net_connections() + assert len(cons) == 1 + for conn in cons: + # TCP v4 + if p.pid == tcp4_proc.pid: + check_conn( + p, + conn, + AF_INET, + SOCK_STREAM, + tcp4_addr, + (), + psutil.CONN_LISTEN, + ("all", "inet", "inet4", "tcp", "tcp4"), + ) + # UDP v4 + elif p.pid == udp4_proc.pid: + check_conn( + p, + conn, + AF_INET, + SOCK_DGRAM, + udp4_addr, + (), + psutil.CONN_NONE, + ("all", "inet", "inet4", "udp", "udp4"), + ) + # TCP v6 + elif p.pid == getattr(tcp6_proc, "pid", None): + check_conn( + p, + conn, + AF_INET6, + SOCK_STREAM, + tcp6_addr, + (), + psutil.CONN_LISTEN, + ("all", "inet", "inet6", "tcp", "tcp6"), + ) + # UDP v6 + elif p.pid == getattr(udp6_proc, "pid", None): + check_conn( + p, + conn, + AF_INET6, + SOCK_DGRAM, + udp6_addr, + (), + psutil.CONN_NONE, + ("all", "inet", "inet6", "udp", "udp6"), + ) + + def test_count(self): + with create_sockets(): + # tcp + cons = this_proc_net_connections(kind='tcp') + assert len(cons) == (2 if supports_ipv6() else 1) + for conn in cons: + assert conn.family in {AF_INET, AF_INET6} + assert conn.type == SOCK_STREAM + # tcp4 + cons = this_proc_net_connections(kind='tcp4') + assert len(cons) == 1 + assert cons[0].family == AF_INET + assert cons[0].type == SOCK_STREAM + # tcp6 + if supports_ipv6(): + cons = this_proc_net_connections(kind='tcp6') + assert len(cons) == 1 + assert cons[0].family == AF_INET6 + assert cons[0].type == SOCK_STREAM + # udp + cons = this_proc_net_connections(kind='udp') + assert len(cons) == (2 if supports_ipv6() else 1) + for conn in cons: + assert conn.family in {AF_INET, AF_INET6} + assert conn.type == SOCK_DGRAM + # udp4 + cons = this_proc_net_connections(kind='udp4') + assert len(cons) == 1 + assert cons[0].family == AF_INET + assert cons[0].type == SOCK_DGRAM + # udp6 + if supports_ipv6(): + cons = this_proc_net_connections(kind='udp6') + assert len(cons) == 1 + assert cons[0].family == AF_INET6 + assert cons[0].type == SOCK_DGRAM + # inet + cons = this_proc_net_connections(kind='inet') + assert len(cons) == (4 if supports_ipv6() else 2) + for conn in cons: + assert conn.family in {AF_INET, AF_INET6} + assert conn.type in {SOCK_STREAM, SOCK_DGRAM} + # inet6 + if supports_ipv6(): + cons = this_proc_net_connections(kind='inet6') + assert len(cons) == 2 + for conn in cons: + assert conn.family == AF_INET6 + assert conn.type in {SOCK_STREAM, SOCK_DGRAM} + # Skipped on BSD because by default the Python process + # creates a UNIX socket to '/var/run/log'. + if HAS_NET_CONNECTIONS_UNIX and not (FREEBSD or NETBSD): + cons = this_proc_net_connections(kind='unix') + assert len(cons) == 3 + for conn in cons: + assert conn.family == AF_UNIX + assert conn.type in {SOCK_STREAM, SOCK_DGRAM} + + +@skipif(SKIP_SYSCONS, reason="requires root") +class TestSystemWideConnections(ConnectionTestCase): + """Tests for net_connections().""" + + def test_it(self): + def check(cons, families, types_): + for conn in cons: + assert conn.family in families + if conn.family != AF_UNIX: + assert conn.type in types_ + check_connection_ntuple(conn) + + with create_sockets(): + from psutil._common import conn_tmap + + for kind, groups in conn_tmap.items(): + # XXX: SunOS does not retrieve UNIX sockets. + if kind == 'unix' and not HAS_NET_CONNECTIONS_UNIX: + continue + families, types_ = groups + cons = psutil.net_connections(kind) + assert len(cons) == len(set(cons)) + check(cons, families, types_) + + @retry_on_failure + def test_multi_sockets_procs(self): + # Creates multiple sub processes, each creating different + # sockets. For each process check that proc.net_connections() + # and psutil.net_connections() return the same results. + # This is done mainly to check whether net_connections()'s + # pid is properly set, see: + # https://github.com/giampaolo/psutil/issues/1013 + with create_sockets() as socks: + expected = len(socks) + procs = [] + times = 10 + fnames = [] + for _ in range(times): + fname = self.get_testfn() + fnames.append(fname) + src = textwrap.dedent(f"""\ + import time, os, sys + if 'CIBUILDWHEEL' not in os.environ: + sys.path.insert(0, r'{ROOT_DIR}') + from tests import create_sockets + with create_sockets(): + with open(r'{fname}', 'w') as f: + f.write("hello") + [time.sleep(0.1) for x in range(100)] + """) + sproc = self.pyrun(src, stderr=subprocess.PIPE) + procs.append(sproc) + + # sync + for fname, sproc in zip(fnames, procs): + wait_for_file_subproc(fname, sproc) + + pids = [x.pid for x in procs] + syscons = [ + x for x in psutil.net_connections(kind='all') if x.pid in pids + ] + for pid in pids: + assert len([x for x in syscons if x.pid == pid]) == expected + p = psutil.Process(pid) + assert len(p.net_connections('all')) == expected + + +class TestMisc(PsutilTestCase): + def test_net_connection_constants(self): + ints = [] + strs = [] + for name in dir(psutil): + if name.startswith('CONN_'): + num = getattr(psutil, name) + str_ = str(num) + assert str_.isupper(), str_ + assert str not in strs + assert num not in ints + ints.append(num) + strs.append(str_) + if SUNOS: + psutil.CONN_IDLE # noqa: B018 + psutil.CONN_BOUND # noqa: B018 + if WINDOWS: + psutil.CONN_DELETE_TCB # noqa: B018 diff --git a/tests/test_contracts.py b/tests/test_contracts.py new file mode 100755 index 0000000000..4d729cc3ad --- /dev/null +++ b/tests/test_contracts.py @@ -0,0 +1,472 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Contracts tests. These tests mainly check API sanity in terms of +returned types and APIs availability. +Some of these are duplicates of tests test_system.py and test_process.py. +""" + +import platform +import socket + +import psutil +from psutil import AIX +from psutil import BSD +from psutil import FREEBSD +from psutil import LINUX +from psutil import MACOS +from psutil import NETBSD +from psutil import OPENBSD +from psutil import POSIX +from psutil import SUNOS +from psutil import WINDOWS +from psutil import BatteryTime +from psutil import ConnectionStatus +from psutil import NicDuplex +from psutil import ProcessStatus + +from . import HAS_CPU_FREQ +from . import HAS_NET_IO_COUNTERS +from . import HAS_SENSORS_FANS +from . import HAS_SENSORS_TEMPERATURES +from . import SKIP_SYSCONS +from . import PsutilTestCase +from . import create_sockets +from . import enum +from . import is_namedtuple +from . import kernel_version +from . import pytest +from . import serial +from . import skipif + +# =================================================================== +# --- APIs availability +# =================================================================== + +# Make sure code reflects what doc promises in terms of APIs +# availability. + + +class TestAvailConstantsAPIs(PsutilTestCase): + + def check_constants(self, names, are_avail): + for name in names: + with self.subTest(name=name): + # assert CONSTANT is/isn't in psutil namespace + assert hasattr(psutil, name) == are_avail + # assert CONSTANT is/isn't in psutil.__all__ + if are_avail: + assert name in psutil.__all__ + else: + assert name not in psutil.__all__ + + def test_PROCFS_PATH(self): + self.check_constants(("PROCFS_PATH",), LINUX or SUNOS or AIX) + + def test_proc_status(self): + names = ( + "STATUS_RUNNING", + "STATUS_SLEEPING", + "STATUS_DISK_SLEEP", + "STATUS_STOPPED", + "STATUS_TRACING_STOP", + "STATUS_ZOMBIE", + "STATUS_DEAD", + "STATUS_WAKE_KILL", + "STATUS_WAKING", + "STATUS_IDLE", + "STATUS_LOCKED", + "STATUS_WAITING", + "STATUS_SUSPENDED", + "STATUS_PARKED", + ) + self.check_constants(names, True) + assert sorted(ProcessStatus.__members__.keys()) == sorted(names) + + def test_proc_status_strenum(self): + mapping = ( + (psutil.STATUS_RUNNING, "running"), + (psutil.STATUS_SLEEPING, "sleeping"), + (psutil.STATUS_DISK_SLEEP, "disk-sleep"), + (psutil.STATUS_STOPPED, "stopped"), + (psutil.STATUS_TRACING_STOP, "tracing-stop"), + (psutil.STATUS_ZOMBIE, "zombie"), + (psutil.STATUS_DEAD, "dead"), + (psutil.STATUS_WAKE_KILL, "wake-kill"), + (psutil.STATUS_WAKING, "waking"), + (psutil.STATUS_IDLE, "idle"), + (psutil.STATUS_LOCKED, "locked"), + (psutil.STATUS_WAITING, "waiting"), + (psutil.STATUS_SUSPENDED, "suspended"), + (psutil.STATUS_PARKED, "parked"), + ) + for en, str_ in mapping: + assert en == str_ + assert str(en) == str_ + assert repr(en) != str_ + + def test_conn_status(self): + names = [ + "CONN_ESTABLISHED", + "CONN_SYN_SENT", + "CONN_SYN_RECV", + "CONN_FIN_WAIT1", + "CONN_FIN_WAIT2", + "CONN_TIME_WAIT", + "CONN_CLOSE", + "CONN_CLOSE_WAIT", + "CONN_LAST_ACK", + "CONN_LISTEN", + "CONN_CLOSING", + "CONN_NONE", + ] + if WINDOWS: + names.append("CONN_DELETE_TCB") + if SUNOS: + names.extend(["CONN_IDLE", "CONN_BOUND"]) + + self.check_constants(names, True) + assert sorted(ConnectionStatus.__members__.keys()) == sorted(names) + + def test_conn_status_strenum(self): + mapping = ( + (psutil.CONN_ESTABLISHED, "ESTABLISHED"), + (psutil.CONN_SYN_SENT, "SYN_SENT"), + (psutil.CONN_SYN_RECV, "SYN_RECV"), + (psutil.CONN_FIN_WAIT1, "FIN_WAIT1"), + (psutil.CONN_FIN_WAIT2, "FIN_WAIT2"), + (psutil.CONN_TIME_WAIT, "TIME_WAIT"), + (psutil.CONN_CLOSE, "CLOSE"), + (psutil.CONN_CLOSE_WAIT, "CLOSE_WAIT"), + (psutil.CONN_LAST_ACK, "LAST_ACK"), + (psutil.CONN_LISTEN, "LISTEN"), + (psutil.CONN_CLOSING, "CLOSING"), + (psutil.CONN_NONE, "NONE"), + ) + for en, str_ in mapping: + assert en == str_ + assert str(en) == str_ + assert repr(en) != str_ + + def test_nic_duplex(self): + names = ("NIC_DUPLEX_FULL", "NIC_DUPLEX_HALF", "NIC_DUPLEX_UNKNOWN") + self.check_constants(names, True) + assert sorted(NicDuplex.__members__.keys()) == sorted(names) + + def test_battery_time(self): + names = ("POWER_TIME_UNKNOWN", "POWER_TIME_UNLIMITED") + self.check_constants(names, True) + assert sorted(BatteryTime.__members__.keys()) == sorted(names) + + def test_proc_ioprio_class_linux(self): + names = ( + "IOPRIO_CLASS_NONE", + "IOPRIO_CLASS_RT", + "IOPRIO_CLASS_BE", + "IOPRIO_CLASS_IDLE", + ) + self.check_constants(names, LINUX) + if LINUX: + assert sorted( + psutil.ProcessIOPriority.__members__.keys() + ) == sorted(names) + else: + not hasattr(psutil, "ProcessIOPriority") + + def test_proc_ioprio_value_windows(self): + names = ( + "IOPRIO_HIGH", + "IOPRIO_NORMAL", + "IOPRIO_LOW", + "IOPRIO_VERYLOW", + ) + self.check_constants(names, WINDOWS) + if WINDOWS: + assert sorted( + psutil.ProcessIOPriority.__members__.keys() + ) == sorted(names) + + def test_proc_priority_windows(self): + names = ( + "ABOVE_NORMAL_PRIORITY_CLASS", + "BELOW_NORMAL_PRIORITY_CLASS", + "HIGH_PRIORITY_CLASS", + "IDLE_PRIORITY_CLASS", + "NORMAL_PRIORITY_CLASS", + "REALTIME_PRIORITY_CLASS", + ) + self.check_constants(names, WINDOWS) + if WINDOWS: + assert sorted(psutil.ProcessPriority.__members__.keys()) == sorted( + names + ) + else: + not hasattr(psutil, "ProcessPriority") + + def test_rlimit(self): + names = ( + "RLIM_INFINITY", + "RLIMIT_AS", + "RLIMIT_CORE", + "RLIMIT_CPU", + "RLIMIT_DATA", + "RLIMIT_FSIZE", + "RLIMIT_MEMLOCK", + "RLIMIT_NOFILE", + "RLIMIT_NPROC", + "RLIMIT_RSS", + "RLIMIT_STACK", + ) + self.check_constants(names, LINUX or FREEBSD) + self.check_constants(("RLIMIT_LOCKS",), LINUX) + self.check_constants( + ("RLIMIT_SWAP", "RLIMIT_SBSIZE", "RLIMIT_NPTS"), FREEBSD + ) + + if POSIX: + if kernel_version() >= (2, 6, 8): + self.check_constants(("RLIMIT_MSGQUEUE",), LINUX) + if kernel_version() >= (2, 6, 12): + self.check_constants(("RLIMIT_NICE", "RLIMIT_RTPRIO"), LINUX) + if kernel_version() >= (2, 6, 25): + self.check_constants(("RLIMIT_RTTIME",), LINUX) + if kernel_version() >= (2, 6, 8): + self.check_constants(("RLIMIT_SIGPENDING",), LINUX) + + def test_enum_containers(self): + self.check_constants(("ProcessStatus",), True) + self.check_constants(("ProcessPriority",), WINDOWS) + self.check_constants(("ProcessIOPriority",), LINUX or WINDOWS) + self.check_constants(("ConnectionStatus",), True) + self.check_constants(("NicDuplex",), True) + self.check_constants(("BatteryTime",), True) + + +class TestAvailSystemAPIs(PsutilTestCase): + def test_win_service_iter(self): + assert hasattr(psutil, "win_service_iter") == WINDOWS + + def test_win_service_get(self): + assert hasattr(psutil, "win_service_get") == WINDOWS + + def test_cpu_freq(self): + assert hasattr(psutil, "cpu_freq") == ( + LINUX or MACOS or WINDOWS or FREEBSD or OPENBSD + ) + + def test_sensors_temperatures(self): + assert hasattr(psutil, "sensors_temperatures") == (LINUX or FREEBSD) + + def test_sensors_fans(self): + assert hasattr(psutil, "sensors_fans") == LINUX + + def test_battery(self): + assert hasattr(psutil, "sensors_battery") == ( + LINUX or WINDOWS or FREEBSD or MACOS + ) + + def test_heap_info(self): + hasit = hasattr(psutil, "heap_info") + if LINUX: + assert hasit == (platform.libc_ver()[0] == "glibc") + else: + assert hasit == MACOS or WINDOWS or BSD + + def test_heap_trim(self): + hasit = hasattr(psutil, "heap_trim") + if LINUX: + assert hasit == (platform.libc_ver()[0] == "glibc") + else: + assert hasit == MACOS or WINDOWS or BSD + + +class TestAvailProcessAPIs(PsutilTestCase): + def test_environ(self): + assert hasattr(psutil.Process, "environ") == ( + LINUX + or MACOS + or WINDOWS + or AIX + or SUNOS + or FREEBSD + or OPENBSD + or NETBSD + ) + + def test_uids(self): + assert hasattr(psutil.Process, "uids") == POSIX + + def test_gids(self): + assert hasattr(psutil.Process, "uids") == POSIX + + def test_terminal(self): + assert hasattr(psutil.Process, "terminal") == POSIX + + def test_ionice(self): + assert hasattr(psutil.Process, "ionice") == (LINUX or WINDOWS) + + def test_rlimit(self): + assert hasattr(psutil.Process, "rlimit") == (LINUX or FREEBSD) + + def test_io_counters(self): + hasit = hasattr(psutil.Process, "io_counters") + assert hasit == (not (MACOS or SUNOS)) + + def test_num_fds(self): + assert hasattr(psutil.Process, "num_fds") == POSIX + + def test_num_handles(self): + assert hasattr(psutil.Process, "num_handles") == WINDOWS + + def test_cpu_affinity(self): + assert hasattr(psutil.Process, "cpu_affinity") == ( + LINUX or WINDOWS or FREEBSD + ) + + def test_cpu_num(self): + assert hasattr(psutil.Process, "cpu_num") == ( + LINUX or FREEBSD or SUNOS + ) + + def test_memory_maps(self): + hasit = hasattr(psutil.Process, "memory_maps") + assert hasit == (not (OPENBSD or NETBSD or AIX or MACOS)) + + def test_memory_extras(self): + hasit = hasattr(psutil.Process, "memory_extras") + assert hasit == (LINUX or MACOS or WINDOWS) + + def test_memory_footprint(self): + hasit = hasattr(psutil.Process, "memory_footprint") + assert hasit == (LINUX or MACOS or WINDOWS) + + +# =================================================================== +# --- API types +# =================================================================== + + +class TestSystemAPITypes(PsutilTestCase): + """Check the return types of system related APIs. + https://github.com/giampaolo/psutil/issues/1039. + """ + + @classmethod + def setUpClass(cls): + cls.proc = psutil.Process() + + def assert_ntuple_of_nums(self, nt, type_=float, gezero=True): + assert is_namedtuple(nt) + for n in nt: + assert isinstance(n, type_) + if gezero: + assert n >= 0 + + def test_cpu_times(self): + self.assert_ntuple_of_nums(psutil.cpu_times()) + for nt in psutil.cpu_times(percpu=True): + self.assert_ntuple_of_nums(nt) + + def test_cpu_percent(self): + assert isinstance(psutil.cpu_percent(interval=None), float) + assert isinstance(psutil.cpu_percent(interval=0.00001), float) + + def test_cpu_times_percent(self): + self.assert_ntuple_of_nums(psutil.cpu_times_percent(interval=None)) + self.assert_ntuple_of_nums(psutil.cpu_times_percent(interval=0.0001)) + + def test_cpu_count(self): + assert isinstance(psutil.cpu_count(), int) + + @skipif(not HAS_CPU_FREQ, reason="not supported") + def test_cpu_freq(self): + if psutil.cpu_freq() is None: + return pytest.skip("cpu_freq() returns None") + self.assert_ntuple_of_nums(psutil.cpu_freq(), type_=(float, int)) + + def test_disk_io_counters(self): + # Duplicate of test_system.py. Keep it anyway. + for k, v in psutil.disk_io_counters(perdisk=True).items(): + assert isinstance(k, str) + self.assert_ntuple_of_nums(v, type_=int) + + def test_disk_partitions(self): + # Duplicate of test_system.py. Keep it anyway. + for disk in psutil.disk_partitions(): + assert isinstance(disk.device, str) + assert isinstance(disk.mountpoint, str) + assert isinstance(disk.fstype, str) + assert isinstance(disk.opts, str) + + @serial + @skipif(SKIP_SYSCONS, reason="requires root") + def test_net_connections(self): + with create_sockets(): + ret = psutil.net_connections('all') + assert len(ret) == len(set(ret)) + for conn in ret: + assert is_namedtuple(conn) + + def test_net_if_addrs(self): + # Duplicate of test_system.py. Keep it anyway. + for ifname, addrs in psutil.net_if_addrs().items(): + assert isinstance(ifname, str) + for addr in addrs: + assert isinstance(addr.family, enum.IntEnum) + assert isinstance(addr.address, (str, type(None))) + if addr.address is None: # virtual NIC + assert addr.family == socket.AF_UNSPEC + assert isinstance(addr.netmask, (str, type(None))) + assert isinstance(addr.broadcast, (str, type(None))) + + def test_net_if_stats(self): + # Duplicate of test_system.py. Keep it anyway. + for ifname, info in psutil.net_if_stats().items(): + assert isinstance(ifname, str) + assert isinstance(info.isup, bool) + assert isinstance(info.duplex, enum.IntEnum) + assert isinstance(info.speed, int) + assert isinstance(info.mtu, int) + + @skipif(not HAS_NET_IO_COUNTERS, reason="not supported") + def test_net_io_counters(self): + # Duplicate of test_system.py. Keep it anyway. + for ifname in psutil.net_io_counters(pernic=True): + assert isinstance(ifname, str) + + @skipif(not HAS_SENSORS_FANS, reason="not supported") + def test_sensors_fans(self): + # Duplicate of test_system.py. Keep it anyway. + for name, units in psutil.sensors_fans().items(): + assert isinstance(name, str) + for unit in units: + assert isinstance(unit.label, str) + assert isinstance(unit.current, (float, int, type(None))) + + @skipif(not HAS_SENSORS_TEMPERATURES, reason="not supported") + def test_sensors_temperatures(self): + # Duplicate of test_system.py. Keep it anyway. + for name, units in psutil.sensors_temperatures().items(): + assert isinstance(name, str) + for unit in units: + assert isinstance(unit.label, str) + assert isinstance(unit.current, (float, int, type(None))) + assert isinstance(unit.high, (float, int, type(None))) + assert isinstance(unit.critical, (float, int, type(None))) + + def test_boot_time(self): + # Duplicate of test_system.py. Keep it anyway. + assert isinstance(psutil.boot_time(), float) + + def test_users(self): + # Duplicate of test_system.py. Keep it anyway. + for user in psutil.users(): + assert isinstance(user.name, str) + assert isinstance(user.terminal, (str, type(None))) + assert isinstance(user.host, (str, type(None))) + assert isinstance(user.pid, (int, type(None))) + if isinstance(user.pid, int): + assert user.pid > 0 diff --git a/tests/test_heap.py b/tests/test_heap.py new file mode 100755 index 0000000000..ab6d1f9ad2 --- /dev/null +++ b/tests/test_heap.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for `psutil.heap_info()`. + +This module deliberately creates **controlled memory leaks** by calling +low-level C allocation functions (`malloc()`, `HeapAlloc()`, +`VirtualAllocEx()`, etc.) **without** freeing them - exactly how +real-world memory leaks occur in native C extensions code. + +By bypassing Python's memory manager entirely (via `ctypes`), we +directly exercise the underlying system allocator: + +UNIX + +- Small `malloc()` allocations (≤ 128KB on glibc) without `free()` + increase `heap_used`. +- Large `malloc()` allocations without `free()` trigger `mmap()` and + increase `mmap_used`. + - Note: direct `mmap()` / `munmap()` via `ctypes` was attempted but + proved unreliable. + +Windows + +- `HeapAlloc()` without `HeapFree()` increases `heap_used`. +- `VirtualAllocEx()` without `VirtualFreeEx()` increases `mmap_used`. +- `HeapCreate()` without `HeapDestroy()` increases `heap_count`. + +These tests ensure that `psutil.heap_info()` detects unreleased +native memory across different allocators (glibc on Linux, +jemalloc on BSD/macOS, Windows CRT). +""" + +import ctypes +import gc + +import psutil +from psutil import LINUX +from psutil import MACOS +from psutil import POSIX +from psutil import WINDOWS + +from . import HAS_HEAP_INFO +from . import PYPY +from . import PsutilTestCase +from . import isolated +from . import retry_on_failure +from . import skipif + +# Small allocation (64 KiB), below M_MMAP_THRESHOLD (128 KiB). +# Increases heap_used (uordblks) without triggering mmap(). +HEAP_SIZE = 64 * 1024 +# Large allocation (64 MiB), exceeds DEFAULT_MMAP_THRESHOLD_MAX (32 +# MiB). Forces malloc() to use mmap() internally and increases +# mmap_used (hblkhd). See `man mallopt`. +MMAP_SIZE = 64 * 1024 * 1024 + + +# ===================================================================== +# --- Utils +# ===================================================================== + + +if POSIX: # noqa: SIM108 + libc = ctypes.CDLL(None) +else: + libc = ctypes.CDLL("msvcrt.dll") + + +def malloc(size): + """Allocate memory via malloc(). If passed a small size, usually + affects heap_used, else mmap_used (not on Windows). + """ + fun = libc.malloc + fun.argtypes = [ctypes.c_size_t] + fun.restype = ctypes.c_void_p + ptr = fun(size) + assert ptr, "malloc() failed" + return ptr + + +def free(ptr): + """Free malloc() memory.""" + fun = libc.free + fun.argtypes = [ctypes.c_void_p] + fun.restype = None + fun(ptr) + + +if WINDOWS: + from ctypes import wintypes + + import win32api + import win32con + import win32process + + kernel32 = ctypes.windll.kernel32 + HEAP_NO_SERIALIZE = 0x00000001 + + # --- for `heap_used` + + def GetProcessHeap(): + fun = kernel32.GetProcessHeap + fun.argtypes = [] + fun.restype = wintypes.HANDLE + heap = fun() + assert heap != 0, "GetProcessHeap failed" + return heap + + def HeapAlloc(heap, size): + fun = kernel32.HeapAlloc + fun.argtypes = [wintypes.HANDLE, wintypes.DWORD, ctypes.c_size_t] + fun.restype = ctypes.c_void_p + addr = fun(heap, 0, size) + assert addr, "HeapAlloc failed" + return addr + + def HeapFree(heap, addr): + fun = kernel32.HeapFree + fun.argtypes = [wintypes.HANDLE, wintypes.DWORD, ctypes.c_void_p] + fun.restype = wintypes.BOOL + assert fun(heap, 0, addr) != 0, "HeapFree failed" + + # --- for `mmap_used` + + def VirtualAllocEx(size): + return win32process.VirtualAllocEx( + win32api.GetCurrentProcess(), + 0, + size, + win32con.MEM_COMMIT | win32con.MEM_RESERVE, + win32con.PAGE_READWRITE, + ) + + def VirtualFreeEx(addr): + win32process.VirtualFreeEx( + win32api.GetCurrentProcess(), addr, 0, win32con.MEM_RELEASE + ) + + # --- for `heap_count` + + def HeapCreate(initial_size, max_size): + fun = kernel32.HeapCreate + fun.argtypes = [ + wintypes.DWORD, + ctypes.c_size_t, + ctypes.c_size_t, + ] + fun.restype = wintypes.HANDLE + heap = fun(HEAP_NO_SERIALIZE, initial_size, max_size) + assert heap != 0, "HeapCreate failed" + return heap + + def HeapDestroy(heap): + fun = kernel32.HeapDestroy + fun.argtypes = [wintypes.HANDLE] + fun.restype = wintypes.BOOL + assert fun(heap) != 0, "HeapDestroy failed" + + +# ===================================================================== +# --- Tests +# ===================================================================== + + +def trim_memory(): + gc.collect() + psutil.heap_trim() + + +def assert_within_percent(actual, expected, percent): + """Assert that `actual` is within `percent` tolerance of `expected`.""" + lower = expected * (1 - percent / 100) + upper = expected * (1 + percent / 100) + if not (lower <= actual <= upper): + raise AssertionError( + f"{actual} is not within {percent}% tolerance of expected" + f" {expected} (allowed range: {lower} - {upper})" + ) + + +@skipif(not HAS_HEAP_INFO, reason="heap_info() not supported") +class HeapTestCase(PsutilTestCase): + def setUp(self): + trim_memory() + + @classmethod + def tearDownClass(cls): + trim_memory() + + +class TestHeap(HeapTestCase): + + # On Windows malloc() increases mmap_used + @skipif(WINDOWS, reason="not on WINDOWS") + @retry_on_failure + def test_heap_used(self): + """Test that a small malloc() allocation without free() + increases heap_used. + """ + size = HEAP_SIZE + + mem1 = psutil.heap_info() + ptr = malloc(size) + mem2 = psutil.heap_info() + + try: + # heap_used should increase (roughly) by the requested size + diff = mem2.heap_used - mem1.heap_used + assert diff > 0 + assert_within_percent(diff, size, percent=10) + + # mmap_used should not increase for small allocations, but + # sometimes it does. + diff = mem2.mmap_used - mem1.mmap_used + if diff != 0: + assert diff > 0 + assert_within_percent(diff, size, percent=10) + finally: + free(ptr) + + # assert we returned close to the baseline (mem1) after free() + trim_memory() + mem3 = psutil.heap_info() + assert_within_percent(mem3.heap_used, mem1.heap_used, percent=10) + assert_within_percent(mem3.mmap_used, mem1.mmap_used, percent=10) + + @skipif(MACOS, reason="not supported on MACOS") + @skipif(PYPY, reason="unstable on PYPY") + @retry_on_failure + def test_mmap_used(self): + """Test that a large malloc allocation increases mmap_used. + NOTE: `mmap()` / `munmap()` via ctypes proved to be unreliable. + """ + size = MMAP_SIZE + + mem1 = psutil.heap_info() + ptr = malloc(size) + mem2 = psutil.heap_info() + + try: + # mmap_used should increase (roughly) by the requested size + diff = mem2.mmap_used - mem1.mmap_used + assert diff > 0 + assert_within_percent(diff, size, percent=10) + + diff = mem2.heap_used - mem1.heap_used + if diff != 0: + if LINUX: + # heap_used should not increase significantly + assert diff >= 0 + assert_within_percent(diff, 0, percent=5) + else: + # On BSD jemalloc allocates big memory both into + # heap_used and mmap_used. + assert_within_percent(diff, size, percent=10) + + finally: + free(ptr) + + # assert we returned close to the baseline (mem1) after free() + trim_memory() + mem3 = psutil.heap_info() + assert_within_percent(mem3.heap_used, mem1.heap_used, percent=10) + assert_within_percent(mem3.mmap_used, mem1.mmap_used, percent=10) + + if WINDOWS: + assert mem1.heap_count == mem2.heap_count == mem3.heap_count + + +@skipif(not WINDOWS, reason="WINDOWS only") +@isolated +class TestHeapWindows(HeapTestCase): + + @retry_on_failure + def test_heap_used(self): + """Test that HeapAlloc() without HeapFree() increases heap_used.""" + size = HEAP_SIZE + + mem1 = psutil.heap_info() + heap = GetProcessHeap() + addr = HeapAlloc(heap, size) + mem2 = psutil.heap_info() + + try: + assert mem2.heap_used - mem1.heap_used == size + finally: + HeapFree(heap, addr) + + trim_memory() + mem3 = psutil.heap_info() + assert mem3.heap_used == mem1.heap_used + + @retry_on_failure + def test_mmap_used(self): + """Test that VirtualAllocEx() without VirtualFreeEx() increases + mmap_used. + """ + size = MMAP_SIZE + + mem1 = psutil.heap_info() + addr = VirtualAllocEx(size) + mem2 = psutil.heap_info() + + try: + assert mem2.mmap_used - mem1.mmap_used == size + finally: + VirtualFreeEx(addr) + + trim_memory() + mem3 = psutil.heap_info() + assert mem3.mmap_used == mem1.mmap_used + + @retry_on_failure + def test_heap_count(self): + """Test that HeapCreate() without HeapDestroy() increases + heap_count. + """ + mem1 = psutil.heap_info() + heap = HeapCreate(HEAP_SIZE, 0) + mem2 = psutil.heap_info() + try: + assert mem2.heap_count == mem1.heap_count + 1 + finally: + HeapDestroy(heap) + + trim_memory() + mem3 = psutil.heap_info() + assert mem3.heap_count == mem1.heap_count diff --git a/tests/test_linux.py b/tests/test_linux.py new file mode 100755 index 0000000000..ada860ce9c --- /dev/null +++ b/tests/test_linux.py @@ -0,0 +1,2651 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Linux specific tests.""" + +import collections +import contextlib +import errno +import glob +import io +import os +import platform +import re +import shutil +import socket +import struct +import textwrap +import time +import warnings +from unittest import mock + +import psutil +from psutil import LINUX +from psutil import _psutil + +from . import AARCH64 +from . import GLOBAL_TIMEOUT +from . import HAS_BATTERY +from . import HAS_CPU_FREQ +from . import HAS_PROC_RLIMIT +from . import TOLERANCE_DISK_USAGE +from . import TOLERANCE_SYS_MEM +from . import PsutilTestCase +from . import ThreadTask +from . import call_until +from . import is_busybox +from . import isolated +from . import pytest +from . import reload_module +from . import requires_cli +from . import retry_on_failure +from . import safe_rmpath +from . import serial +from . import sh +from . import skip_on_not_implemented +from . import skipif + +if LINUX: + from psutil._pslinux import CLOCK_TICKS + from psutil._pslinux import RootFsDeviceFinder + from psutil._pslinux import _cpu_get_cpuinfo_freq + from psutil._pslinux import _parse_cpulist + from psutil._pslinux import calculate_avail_vmem + from psutil._pslinux import open_binary + + +SIOCGIFADDR = 0x8915 +SIOCGIFHWADDR = 0x8927 +SIOCGIFNETMASK = 0x891B +SIOCGIFBRDADDR = 0x8919 +if LINUX: + SECTOR_SIZE = 512 +# Overlayfs and btrfs give / an anonymous device (major 0), which has +# no /proc/partitions or /sys/dev/block entry to look up. +ROOTFS_ON_BLOCK_DEV = LINUX and os.major(os.stat("/").st_dev) != 0 +CPUINFO_HAS_MHZ = LINUX and bool(_cpu_get_cpuinfo_freq()) + + +@skipif(not LINUX, reason="LINUX only") +class LinuxTestCase(PsutilTestCase): + pass + + +# ===================================================================== +# --- utils +# ===================================================================== + + +def get_ipv4_address(ifname): + import fcntl + + ifname = bytes(ifname[:15], "ascii") + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + return socket.inet_ntoa( + fcntl.ioctl(s.fileno(), SIOCGIFADDR, struct.pack('256s', ifname))[ + 20:24 + ] + ) + + +def get_ipv4_netmask(ifname): + import fcntl + + ifname = bytes(ifname[:15], "ascii") + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + return socket.inet_ntoa( + fcntl.ioctl( + s.fileno(), SIOCGIFNETMASK, struct.pack('256s', ifname) + )[20:24] + ) + + +def get_ipv4_broadcast(ifname): + import fcntl + + ifname = bytes(ifname[:15], "ascii") + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + return socket.inet_ntoa( + fcntl.ioctl( + s.fileno(), SIOCGIFBRDADDR, struct.pack('256s', ifname) + )[20:24] + ) + + +def get_ipv6_addresses(ifname): + with open("/proc/net/if_inet6") as f: + all_fields = [] + for line in f: + fields = line.split() + if fields[-1] == ifname: + all_fields.append(fields) + + if len(all_fields) == 0: + raise ValueError(f"could not find interface {ifname!r}") + + for i in range(len(all_fields)): + unformatted = all_fields[i][0] + groups = [ + unformatted[j : j + 4] for j in range(0, len(unformatted), 4) + ] + formatted = ":".join(groups) + packed = socket.inet_pton(socket.AF_INET6, formatted) + all_fields[i] = socket.inet_ntop(socket.AF_INET6, packed) + return all_fields + + +def get_mac_address(ifname): + import fcntl + + ifname = bytes(ifname[:15], "ascii") + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + info = fcntl.ioctl( + s.fileno(), SIOCGIFHWADDR, struct.pack('256s', ifname) + ) + return "".join([f"{char:02x}:" for char in info[18:24]])[:-1] + + +def free_swap(): + """Parse 'free' cmd and return swap memory's s total, used and free + values. + """ + out = sh(["free", "-b"], env={"LANG": "C.UTF-8"}) + lines = out.split('\n') + for line in lines: + if line.startswith('Swap'): + _, total, used, free = line.split() + nt = collections.namedtuple('free', 'total used free') + return nt(int(total), int(used), int(free)) + raise ValueError(f"can't find 'Swap' in 'free' output:\n{out}") + + +def free_physmem(): + """Parse 'free' cmd and return physical memory's total, used + and free values. + """ + # Note: free can have 2 different formats, invalidating 'shared' + # and 'cached' memory which may have different positions so we + # do not return them. + # https://github.com/giampaolo/psutil/issues/538#issuecomment-57059946 + out = sh(["free", "-b"], env={"LANG": "C.UTF-8"}) + lines = out.split('\n') + for line in lines: + if line.startswith('Mem'): + total, used, free, shared = (int(x) for x in line.split()[1:5]) + nt = collections.namedtuple( + 'free', 'total used free shared output' + ) + return nt(total, used, free, shared, out) + raise ValueError(f"can't find 'Mem' in 'free' output:\n{out}") + + +@requires_cli("vmstat") +def vmstat(stat): + out = sh(["vmstat", "-s"], env={"LANG": "C.UTF-8"}) + for line in out.split("\n"): + line = line.strip() + if stat in line: + return int(line.split(' ')[0]) + raise ValueError(f"can't find {stat!r} in 'vmstat' output") + + +def get_free_version_info(): + if is_busybox("free"): + return pytest.skip("busybox free has no -V option") + out = sh(["free", "-V"]).strip() + if 'UNKNOWN' in out: + return pytest.skip("can't determine free version") + return tuple(map(int, re.findall(r'\d+', out.split()[-1]))) + + +@contextlib.contextmanager +def mock_open_content(pairs): + """Mock open() builtin and forces it to return a certain content + for a given path. `pairs` is a {"path": "content", ...} dict. + """ + + def open_mock(name, *args, **kwargs): + if name in pairs: + content = pairs[name] + if isinstance(content, str): + return io.StringIO(content) + else: + return io.BytesIO(content) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", create=True, side_effect=open_mock) as m: + yield m + + +@contextlib.contextmanager +def mock_open_exception(for_path, exc): + """Mock open() builtin and raises `exc` if the path being opened + matches `for_path`. + """ + + def open_mock(name, *args, **kwargs): + if name == for_path: + raise exc + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", create=True, side_effect=open_mock) as m: + yield m + + +# ===================================================================== +# --- system virtual memory +# ===================================================================== + + +class TestVirtualMemoryAgainstFree(LinuxTestCase): + def test_total(self): + cli_value = free_physmem().total + psutil_value = psutil.virtual_memory().total + assert cli_value == psutil_value + + @retry_on_failure + def test_used(self): + # Older versions of procps used slab memory to calculate used memory. + # This got changed in: + # https://gitlab.com/procps-ng/procps/-/commit/05d751c4f07 + # Newer versions of procps (>=4.0.1) are using yet another way to + # compute used memory. + # https://gitlab.com/procps-ng/procps/-/commit/2184e90d2e7 + if get_free_version_info() < (4, 0, 1): + return pytest.skip("free version too old") + cli_value = free_physmem().used + psutil_value = psutil.virtual_memory().used + assert abs(cli_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_free(self): + cli_value = free_physmem().free + psutil_value = psutil.virtual_memory().free + assert abs(cli_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_shared(self): + free = free_physmem() + free_value = free.shared + if free_value == 0: + return pytest.skip("free does not support 'shared' column") + psutil_value = psutil.virtual_memory().shared + assert ( + abs(free_value - psutil_value) < TOLERANCE_SYS_MEM + ), f"{free_value} {psutil_value} \n{free.output}" + + @retry_on_failure + def test_available(self): + # "free" output format has changed at some point: + # https://github.com/giampaolo/psutil/issues/538#issuecomment-147192098 + out = sh(["free", "-b"]) + lines = out.split('\n') + if 'available' not in lines[0]: + return pytest.skip("free does not support 'available' column") + free_value = int(lines[1].split()[-1]) + psutil_value = psutil.virtual_memory().available + assert abs(free_value - psutil_value) < TOLERANCE_SYS_MEM + + +class TestVirtualMemoryAgainstVmstat(LinuxTestCase): + def test_total(self): + vmstat_value = vmstat('total memory') * 1024 + psutil_value = psutil.virtual_memory().total + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_used(self): + # Older versions of procps used slab memory to calculate used memory. + # This got changed in: + # https://gitlab.com/procps-ng/procps/-/commit/05d751c4f07 + # Newer versions of procps (>=4.0.1) are using yet another way to + # compute used memory. + # https://gitlab.com/procps-ng/procps/-/commit/2184e90d2e7 + if get_free_version_info() < (4, 0, 1): + return pytest.skip("free version too old") + vmstat_value = vmstat('used memory') * 1024 + psutil_value = psutil.virtual_memory().used + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_free(self): + vmstat_value = vmstat('free memory') * 1024 + psutil_value = psutil.virtual_memory().free + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_buffers(self): + vmstat_value = vmstat('buffer memory') * 1024 + psutil_value = psutil.virtual_memory().buffers + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_active(self): + vmstat_value = vmstat('active memory') * 1024 + psutil_value = psutil.virtual_memory().active + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_inactive(self): + vmstat_value = vmstat('inactive memory') * 1024 + psutil_value = psutil.virtual_memory().inactive + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + +class TestVirtualMemoryAgainstMeminfo(LinuxTestCase): + @staticmethod + def read_meminfo(): + mems = {} + with open("/proc/meminfo") as f: + for line in f: + fields = line.split() + if len(fields) >= 2: + mems[fields[0]] = int(fields[1]) * 1024 + return mems + + @retry_on_failure + def test_buffers(self): + proc_value = self.read_meminfo()["Buffers:"] + psutil_value = psutil.virtual_memory().buffers + assert abs(psutil_value - proc_value) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_cached(self): + # psutil cached = Cached + SReclaimable + mems = self.read_meminfo() + proc_value = mems["Cached:"] + mems.get("SReclaimable:", 0) + psutil_value = psutil.virtual_memory().cached + assert abs(psutil_value - proc_value) < TOLERANCE_SYS_MEM + + +class TestVirtualMemoryMocks(LinuxTestCase): + def test_warnings_on_misses(self): + # Emulate a case where /proc/meminfo provides few info. + # psutil is supposed to set the missing fields to 0 and + # raise a warning. + content = textwrap.dedent("""\ + Active(anon): 6145416 kB + Active(file): 2950064 kB + Inactive(anon): 574764 kB + Inactive(file): 1567648 kB + MemAvailable: -1 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + SReclaimable: 346648 kB + """).encode() + with mock_open_content({'/proc/meminfo': content}) as m: + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") + ret = psutil.virtual_memory() + assert m.called + assert len(ws) == 1 + w = ws[0] + assert "memory stats couldn't be determined" in str(w.message) + assert "cached" in str(w.message) + assert "shared" in str(w.message) + assert "active" in str(w.message) + assert "inactive" in str(w.message) + assert "buffers" in str(w.message) + assert "available" in str(w.message) + assert ret.cached == 0 + assert ret.active == 0 + assert ret.inactive == 0 + assert ret.shared == 0 + assert ret.buffers == 0 + assert ret.available == 0 + assert ret.slab == 0 + + @retry_on_failure + def test_avail_old_percent(self): + # Make sure that our calculation of avail mem for old kernels + # is off by max 15%. + mems = {} + with open_binary('/proc/meminfo') as f: + for line in f: + fields = line.split() + mems[fields[0]] = int(fields[1]) * 1024 + + a = calculate_avail_vmem(mems) + if b'MemAvailable:' in mems: + b = mems[b'MemAvailable:'] + diff_percent = abs(a - b) / a * 100 + assert diff_percent < 15 + + def test_avail_old_comes_from_kernel(self): + # Make sure "MemAvailable:" coluimn is used instead of relying + # on our internal algorithm to calculate avail mem. + content = textwrap.dedent("""\ + Active: 9444728 kB + Active(anon): 6145416 kB + Active(file): 2950064 kB + Buffers: 287952 kB + Cached: 4818144 kB + Inactive(file): 1578132 kB + Inactive(anon): 574764 kB + Inactive(file): 1567648 kB + MemAvailable: 6574984 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + Shmem: 577588 kB + SReclaimable: 346648 kB + """).encode() + with mock_open_content({'/proc/meminfo': content}) as m: + with warnings.catch_warnings(record=True) as ws: + ret = psutil.virtual_memory() + assert m.called + assert ret.available == 6574984 * 1024 + w = ws[0] + assert "inactive memory stats couldn't be determined" in str( + w.message + ) + + def test_avail_old_missing_fields(self): + # Remove Active(file), Inactive(file) and SReclaimable + # from /proc/meminfo and make sure the fallback is used + # (free + cached), + content = textwrap.dedent("""\ + Active: 9444728 kB + Active(anon): 6145416 kB + Buffers: 287952 kB + Cached: 4818144 kB + Inactive(file): 1578132 kB + Inactive(anon): 574764 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + Shmem: 577588 kB + """).encode() + with mock_open_content({"/proc/meminfo": content}) as m: + with warnings.catch_warnings(record=True) as ws: + ret = psutil.virtual_memory() + assert m.called + assert ret.available == 2057400 * 1024 + 4818144 * 1024 + w = ws[0] + assert "inactive memory stats couldn't be determined" in str( + w.message + ) + + def test_avail_old_missing_zoneinfo(self): + # Remove /proc/zoneinfo file. Make sure fallback is used + # (free + cached). + content = textwrap.dedent("""\ + Active: 9444728 kB + Active(anon): 6145416 kB + Active(file): 2950064 kB + Buffers: 287952 kB + Cached: 4818144 kB + Inactive(file): 1578132 kB + Inactive(anon): 574764 kB + Inactive(file): 1567648 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + Shmem: 577588 kB + SReclaimable: 346648 kB + """).encode() + with mock_open_content({"/proc/meminfo": content}): + with mock_open_exception("/proc/zoneinfo", FileNotFoundError): + with warnings.catch_warnings(record=True) as ws: + ret = psutil.virtual_memory() + assert ret.available == 2057400 * 1024 + 4818144 * 1024 + w = ws[0] + assert ( + "inactive memory stats couldn't be determined" + in str(w.message) + ) + + def test_virtual_memory_mocked(self): + # Emulate /proc/meminfo because neither vmstat nor free return slab. + content = textwrap.dedent("""\ + MemTotal: 100 kB + MemFree: 2 kB + MemAvailable: 3 kB + Buffers: 4 kB + Cached: 5 kB + SwapCached: 6 kB + Active: 7 kB + Inactive: 8 kB + Active(anon): 9 kB + Inactive(anon): 10 kB + Active(file): 11 kB + Inactive(file): 12 kB + Unevictable: 13 kB + Mlocked: 14 kB + SwapTotal: 15 kB + SwapFree: 16 kB + Dirty: 17 kB + Writeback: 18 kB + AnonPages: 19 kB + Mapped: 20 kB + Shmem: 21 kB + Slab: 22 kB + SReclaimable: 23 kB + SUnreclaim: 24 kB + KernelStack: 25 kB + PageTables: 26 kB + NFS_Unstable: 27 kB + Bounce: 28 kB + WritebackTmp: 29 kB + CommitLimit: 30 kB + Committed_AS: 31 kB + VmallocTotal: 32 kB + VmallocUsed: 33 kB + VmallocChunk: 34 kB + HardwareCorrupted: 35 kB + AnonHugePages: 36 kB + ShmemHugePages: 37 kB + ShmemPmdMapped: 38 kB + CmaTotal: 39 kB + CmaFree: 40 kB + HugePages_Total: 41 kB + HugePages_Free: 42 kB + HugePages_Rsvd: 43 kB + HugePages_Surp: 44 kB + Hugepagesize: 45 kB + DirectMap46k: 46 kB + DirectMap47M: 47 kB + DirectMap48G: 48 kB + """).encode() + with mock_open_content({"/proc/meminfo": content}) as m: + mem = psutil.virtual_memory() + assert m.called + assert mem.total == 100 * 1024 + assert mem.free == 2 * 1024 + assert mem.buffers == 4 * 1024 + # cached mem also includes reclaimable memory + assert mem.cached == (5 + 23) * 1024 + assert mem.shared == 21 * 1024 + assert mem.active == 7 * 1024 + assert mem.inactive == 8 * 1024 + assert mem.slab == 22 * 1024 + assert mem.available == 3 * 1024 + + def test_virtual_memory_no_space_after_colon(self): + """Some Linux meminfo fields may not have a space after the colon: + https://github.com/torvalds/linux/blob/8356a5a3b078ca89c526dd6d71e9a76fec571c37/fs/proc/meminfo.c#L113-L116 + """ + content = textwrap.dedent("""\ + MemTotal: 100 kB + MemFree: 2 kB + MemAvailable: 3 kB + Buffers: 4 kB + Cached: 5 kB + Active: 6 kB + Inactive: 7 kB + Shmem: 8 kB + Slab: 9 kB + ShadowCallStack:10373888 kB + """).encode() + with mock_open_content({"/proc/meminfo": content}) as m: + mem = psutil.virtual_memory() + assert m.called + assert mem.total == 100 * 1024 + + +# ===================================================================== +# --- system swap memory +# ===================================================================== + + +class TestSwapMemory(LinuxTestCase): + @staticmethod + def meminfo_has_swap_info(): + """Return True if /proc/meminfo provides swap metrics.""" + with open("/proc/meminfo") as f: + data = f.read() + return 'SwapTotal:' in data and 'SwapFree:' in data + + def test_total(self): + free_value = free_swap().total + psutil_value = psutil.swap_memory().total + assert abs(free_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_used(self): + free_value = free_swap().used + psutil_value = psutil.swap_memory().used + assert abs(free_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_free(self): + free_value = free_swap().free + psutil_value = psutil.swap_memory().free + assert abs(free_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_sin_sout(self): + # Cross-check sin/sout against /proc/vmstat pswpin/pswpout fields. + # psutil converts pages to bytes using a 4096-byte page size. + PAGE_SIZE = 4 * 1024 + with open("/proc/vmstat") as f: + vmstat = dict(line.split() for line in f if line.split()) + sin = int(vmstat["pswpin"]) * PAGE_SIZE + sout = int(vmstat["pswpout"]) * PAGE_SIZE + swap = psutil.swap_memory() + assert swap.sin == sin + assert swap.sout == sout + + def test_missing_sin_sout(self): + with mock.patch('psutil._common.open', create=True) as m: + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") + ret = psutil.swap_memory() + assert m.called + assert len(ws) == 1 + w = ws[0] + assert ( + "'sin' and 'sout' swap memory stats couldn't be determined" + in str(w.message) + ) + assert ret.sin == 0 + assert ret.sout == 0 + + def test_no_vmstat_mocked(self): + # see https://github.com/giampaolo/psutil/issues/722 + with mock_open_exception("/proc/vmstat", FileNotFoundError) as m: + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") + ret = psutil.swap_memory() + assert m.called + assert len(ws) == 1 + w = ws[0] + assert ( + "'sin' and 'sout' swap memory stats couldn't " + "be determined and were set to 0" + in str(w.message) + ) + assert ret.sin == 0 + assert ret.sout == 0 + + def test_meminfo_against_sysinfo(self): + # Make sure the content of /proc/meminfo about swap memory + # matches sysinfo() syscall, see: + # https://github.com/giampaolo/psutil/issues/1015 + if not self.meminfo_has_swap_info(): + return pytest.skip("/proc/meminfo has no swap metrics") + with mock.patch.object(_psutil, 'linux_sysinfo') as m: + swap = psutil.swap_memory() + assert not m.called + + _, _, _, _, total, free, unit_multiplier = _psutil.linux_sysinfo() + total *= unit_multiplier + free *= unit_multiplier + assert swap.total == total + assert abs(swap.free - free) < TOLERANCE_SYS_MEM + + def test_emulate_meminfo_has_no_metrics(self): + # Emulate a case where /proc/meminfo provides no swap metrics + # in which case sysinfo() syscall is supposed to be used + # as a fallback. + with mock_open_content({"/proc/meminfo": b""}) as m: + psutil.swap_memory() + assert m.called + + def test_no_space_after_colon(self): + # Some Linux meminfo fields may not have a space after the + # colon, see: + # https://github.com/giampaolo/psutil/issues/2809 + content = textwrap.dedent("""\ + MemTotal: 100 kB + MemFree: 2 kB + SwapTotal: 15 kB + SwapFree: 14 kB + ShadowCallStack:10373888 kB + """).encode() + with mock_open_content({"/proc/meminfo": content}) as m: + swap = psutil.swap_memory() + assert m.called + assert swap.total == 15 * 1024 + assert swap.free == 14 * 1024 + + +# ===================================================================== +# --- system CPU +# ===================================================================== + + +class TestCpuCountLogical(LinuxTestCase): + @skipif( + not os.path.exists("/sys/devices/system/cpu/online"), + reason="/sys/devices/system/cpu/online does not exist", + ) + def test_against_sysdev_cpu_online(self): + with open("/sys/devices/system/cpu/online") as f: + value = f.read().strip() + if "-" in str(value): + value = int(value.split('-')[1]) + 1 + assert psutil.cpu_count() == value + + @skipif( + not os.path.exists("/sys/devices/system/cpu"), + reason="/sys/devices/system/cpu does not exist", + ) + def test_against_sysdev_cpu_num(self): + # The kernel creates a directory for every *possible* CPU, so + # this is an upper bound: cpu_count() counts the online ones. + ls = os.listdir("/sys/devices/system/cpu") + count = len([x for x in ls if re.search(r"cpu\d+$", x) is not None]) + assert psutil.cpu_count() <= count + + @requires_cli("nproc") + def test_against_nproc(self): + num = int(sh("nproc")) + assert psutil.cpu_count(logical=True) == num + + @requires_cli("lscpu") + def test_against_lscpu(self): + out = sh("lscpu -p") + num = len([x for x in out.split('\n') if not x.startswith('#')]) + assert psutil.cpu_count(logical=True) == num + + def test_emulate_fallbacks(self): + import psutil._pslinux + + original = psutil._pslinux.cpu_count_logical() + # Here we want to mock os.sysconf("SC_NPROCESSORS_ONLN") in + # order to cause the parsing of /proc/cpuinfo and /proc/stat. + with mock.patch( + 'psutil._pslinux.os.sysconf', side_effect=ValueError + ) as m: + assert psutil._pslinux.cpu_count_logical() == original + assert m.called + + # Let's have open() return empty data and make sure None is + # returned ('cause we mimic os.cpu_count()). + with mock.patch('psutil._common.open', create=True) as m: + assert psutil._pslinux.cpu_count_logical() is None + assert m.call_count == 2 + # /proc/stat should be the last one + assert m.call_args[0][0] == '/proc/stat' + + # Let's push this a bit further and make sure /proc/cpuinfo + # parsing works as expected. + with open('/proc/cpuinfo', 'rb') as f: + cpuinfo_data = f.read() + with mock_open_content({"/proc/cpuinfo": cpuinfo_data}) as m: + assert psutil._pslinux.cpu_count_logical() == original + assert m.called + + # Finally, let's make /proc/cpuinfo return meaningless data; + # this way we'll fall back on relying on /proc/stat + with mock_open_content({"/proc/cpuinfo": b""}) as m: + assert psutil._pslinux.cpu_count_logical() == original + assert m.called + + +class TestCpuCountCores(LinuxTestCase): + @requires_cli("lscpu") + def test_against_lscpu(self): + pairs = [] + per_socket = "" + for line in sh("lscpu").splitlines(): + key, _, value = line.partition(":") + key, value = key.strip(), value.strip() + if key == "Core(s) per socket": + per_socket = value + elif key == "Socket(s)": + pairs.append((per_socket, value)) + # In a VM lscpu may print "-" instead of a number. + if not pairs or not all(x.isdigit() and y.isdigit() for x, y in pairs): + return pytest.skip("lscpu doesn't report the number of cores") + cores = sum(int(x) * int(y) for x, y in pairs) + assert psutil.cpu_count(logical=False) == cores + + @skipif( + platform.machine() not in {"x86_64", "i686"}, reason="x86_64/i686 only" + ) + def test_method_2(self): + meth_1 = psutil._pslinux.cpu_count_cores() + with mock.patch('glob.glob', return_value=[]) as m: + meth_2 = psutil._pslinux.cpu_count_cores() + assert m.called + if meth_1 is not None: + assert meth_1 == meth_2 + + def test_emulate_none(self): + with mock.patch('glob.glob', return_value=[]) as m1: + with mock.patch('psutil._common.open', create=True) as m2: + assert psutil._pslinux.cpu_count_cores() is None + assert m1.called + assert m2.called + + +class TestCpuFreq(LinuxTestCase): + def test_cpuinfo_freq_ppc(self): + content = b"clock\t\t: 2750.000000MHz\nclock\t\t: 2500.000000MHz\n" + with mock_open_content({"/proc/cpuinfo": content}): + assert _cpu_get_cpuinfo_freq() == [2750.0, 2500.0] + + def test_cpuinfo_freq_s390x(self): + content = ( + b"cpu MHz dynamic : 5200\ncpu MHz static : 5000\n" + b"cpu MHz dynamic : 5100\ncpu MHz static : 5000\n" + ) + with mock_open_content({"/proc/cpuinfo": content}): + assert _cpu_get_cpuinfo_freq() == [5200.0, 5100.0] + + @skipif(not HAS_CPU_FREQ, reason="not supported") + @skipif(AARCH64, reason="aarch64 does not always expose frequency") + def test_emulate_use_second_file(self): + # https://github.com/giampaolo/psutil/issues/981 + def path_exists_mock(path): + if path.startswith("/sys/devices/system/cpu/cpufreq/policy"): + return False + else: + return orig_exists(path) + + orig_exists = os.path.exists + with mock.patch( + "os.path.exists", side_effect=path_exists_mock, create=True + ): + assert psutil.cpu_freq() + + @skipif(not HAS_CPU_FREQ, reason="not supported") + @skipif(not CPUINFO_HAS_MHZ, reason="no 'cpu MHz' in /proc/cpuinfo") + def test_emulate_use_cpuinfo(self): + # Emulate a case where /sys/devices/system/cpu/cpufreq* does not + # exist and /proc/cpuinfo is used instead. + def path_exists_mock(path): + if path.startswith('/sys/devices/system/cpu/'): + return False + else: + return os_path_exists(path) + + os_path_exists = os.path.exists + try: + with mock.patch("os.path.exists", side_effect=path_exists_mock): + reload_module(psutil._pslinux) + ret = psutil.cpu_freq() + assert ret, ret + assert ret.max == 0.0 + assert ret.min == 0.0 + for freq in psutil.cpu_freq(percpu=True): + assert freq.max == 0.0 + assert freq.min == 0.0 + finally: + reload_module(psutil._pslinux) + reload_module(psutil) + + @skipif(not HAS_CPU_FREQ, reason="not supported") + def test_emulate_data(self): + def open_mock(name, *args, **kwargs): + if name.endswith('/scaling_cur_freq') and name.startswith( + "/sys/devices/system/cpu/cpufreq/policy" + ): + return io.BytesIO(b"500000") + elif name.endswith('/scaling_min_freq') and name.startswith( + "/sys/devices/system/cpu/cpufreq/policy" + ): + return io.BytesIO(b"600000") + elif name.endswith('/scaling_max_freq') and name.startswith( + "/sys/devices/system/cpu/cpufreq/policy" + ): + return io.BytesIO(b"700000") + elif name == '/proc/cpuinfo': + return io.BytesIO(b"cpu MHz : 500") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock): + with mock.patch('os.path.exists', return_value=True): + freq = psutil.cpu_freq() + assert freq.current == 500.0 + # when /proc/cpuinfo is used min and max frequencies are not + # available and are set to 0. + if freq.min != 0.0: + assert freq.min == 600.0 + if freq.max != 0.0: + assert freq.max == 700.0 + + @skipif(not HAS_CPU_FREQ, reason="not supported") + def test_emulate_multi_cpu(self): + def open_mock(name, *args, **kwargs): + n = name + if n.endswith('/scaling_cur_freq') and n.startswith( + "/sys/devices/system/cpu/cpufreq/policy0" + ): + return io.BytesIO(b"100000") + elif n.endswith('/scaling_min_freq') and n.startswith( + "/sys/devices/system/cpu/cpufreq/policy0" + ): + return io.BytesIO(b"200000") + elif n.endswith('/scaling_max_freq') and n.startswith( + "/sys/devices/system/cpu/cpufreq/policy0" + ): + return io.BytesIO(b"300000") + elif n.endswith('/scaling_cur_freq') and n.startswith( + "/sys/devices/system/cpu/cpufreq/policy1" + ): + return io.BytesIO(b"400000") + elif n.endswith('/scaling_min_freq') and n.startswith( + "/sys/devices/system/cpu/cpufreq/policy1" + ): + return io.BytesIO(b"500000") + elif n.endswith('/scaling_max_freq') and n.startswith( + "/sys/devices/system/cpu/cpufreq/policy1" + ): + return io.BytesIO(b"600000") + elif n.endswith('/affected_cpus'): + return io.BytesIO(b"0" if "/policy0/" in n else b"1") + elif name == '/proc/cpuinfo': + return io.BytesIO(b"cpu MHz : 100\ncpu MHz : 400") + else: + return orig_open(name, *args, **kwargs) + + def glob_mock(pattern): + if pattern == "/sys/devices/system/cpu/cpufreq/policy[0-9]*": + return list(policies) + return orig_glob(pattern) + + policies = [ + f"/sys/devices/system/cpu/cpufreq/policy{n}" for n in range(2) + ] + orig_glob = glob.glob + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock): + with mock.patch('os.path.exists', return_value=True): + with mock.patch("glob.glob", side_effect=glob_mock): + # min and max are 0 when the /proc/cpuinfo-only + # implementation is in use, e.g. in a container + # with no /sys/devices/system/cpu/cpufreq. + freq = psutil.cpu_freq(percpu=True) + assert freq[0].current == 100.0 + if freq[0].min != 0.0: + assert freq[0].min == 200.0 + if freq[0].max != 0.0: + assert freq[0].max == 300.0 + assert freq[1].current == 400.0 + if freq[1].min != 0.0: + assert freq[1].min == 500.0 + if freq[1].max != 0.0: + assert freq[1].max == 600.0 + + @skipif(not HAS_CPU_FREQ, reason="not supported") + def test_emulate_shared_policy(self): + # A single policy governing 4 CPUs must yield 4 entries, see: + # https://github.com/giampaolo/psutil/issues/2512 + def open_mock(name, *args, **kwargs): + if name.endswith('/affected_cpus'): + return io.BytesIO(b"0 1 2 3") + elif name.endswith('/scaling_cur_freq'): + return io.BytesIO(b"100000") + elif name.endswith('/scaling_min_freq'): + return io.BytesIO(b"200000") + elif name.endswith('/scaling_max_freq'): + return io.BytesIO(b"300000") + elif name == '/proc/cpuinfo': + return io.BytesIO(b"") + return orig_open(name, *args, **kwargs) + + def glob_mock(pattern): + if pattern == "/sys/devices/system/cpu/cpufreq/policy[0-9]*": + return ["/sys/devices/system/cpu/cpufreq/policy0"] + return orig_glob(pattern) + + def exists_mock(path): + # Make sure the sysfs-based implementation is used. + if path.startswith("/sys/devices/system/cpu/"): + return True + return orig_exists(path) + + orig_exists = os.path.exists + orig_glob = glob.glob + orig_open = open + try: + with mock.patch("os.path.exists", side_effect=exists_mock): + reload_module(psutil._pslinux) + with mock.patch("glob.glob", side_effect=glob_mock): + with mock.patch("builtins.open", side_effect=open_mock): + freq = psutil.cpu_freq(percpu=True) + assert len(freq) == 4 + for nt in freq: + assert nt == (100.0, 200.0, 300.0) + finally: + reload_module(psutil._pslinux) + reload_module(psutil) + + @skipif(not HAS_CPU_FREQ, reason="not supported") + def test_emulate_no_scaling_cur_freq_file(self): + # See: https://github.com/giampaolo/psutil/issues/1071 + def open_mock(name, *args, **kwargs): + if name.endswith('/scaling_cur_freq'): + raise FileNotFoundError + if name.endswith('/cpuinfo_cur_freq'): + return io.BytesIO(b"200000") + elif name == '/proc/cpuinfo': + return io.BytesIO(b"cpu MHz : 200") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock): + with mock.patch('os.path.exists', return_value=True): + with mock.patch( + 'psutil._pslinux.cpu_count_logical', return_value=1 + ): + freq = psutil.cpu_freq() + assert freq.current == 200 + + @skipif(not HAS_CPU_FREQ, reason="not supported") + def test_emulate_offline_cpus(self): + # Offline CPU cores must not be taken into account, else they + # drag down the average frequency. See: + # https://github.com/giampaolo/psutil/issues/2628 + policies = [ + f"/sys/devices/system/cpu/cpufreq/policy{n}" for n in range(4) + ] + + def exists_mock(path): + # Make sure the sysfs-based implementation is used. + if path.startswith("/sys/devices/system/cpu/"): + return True + return orig_exists(path) + + def glob_mock(pattern): + if pattern == "/sys/devices/system/cpu/cpufreq/policy[0-9]*": + return list(policies) + return orig_glob(pattern) + + def open_mock(name, *args, **kwargs): + # Only CPUs 0 and 1 are online; 2 and 3 are offline. + if name == '/proc/cpuinfo': + return io.BytesIO(b"cpu MHz\t: 200\ncpu MHz\t: 400") + elif name.endswith('/affected_cpus'): + num = re.search(r"/policy(\d+)/", name).group(1) + return io.BytesIO(num.encode()) + elif "/policy0/" in name or "/policy1/" in name: + if name.endswith('/scaling_cur_freq'): + cur = b"200000" if "/policy0/" in name else b"400000" + return io.BytesIO(cur) + elif name.endswith('/scaling_min_freq'): + return io.BytesIO(b"100000") + elif name.endswith('/scaling_max_freq'): + return io.BytesIO(b"300000") + elif "/policy2/" in name or "/policy3/" in name: + # Offline cores have no frequency files. + if name.endswith(('/scaling_cur_freq', '/cpuinfo_cur_freq')): + raise FileNotFoundError + elif name.endswith('/online'): + # CPUs 2 and 3 are offline. + return io.StringIO("0\n") + return orig_open(name, *args, **kwargs) + + orig_exists = os.path.exists + orig_glob = glob.glob + orig_open = open + try: + with mock.patch("os.path.exists", side_effect=exists_mock): + reload_module(psutil._pslinux) + with mock.patch("glob.glob", side_effect=glob_mock): + with mock.patch("builtins.open", side_effect=open_mock): + percpu = psutil.cpu_freq(percpu=True) + assert len(percpu) == 2 + assert [f.current for f in percpu] == [200.0, 400.0] + + freq = psutil.cpu_freq() + assert freq.current == 300.0 + assert freq.min == 100.0 + assert freq.max == 300.0 + finally: + reload_module(psutil._pslinux) + reload_module(psutil) + + +class TestCpuTimes(LinuxTestCase): + + @retry_on_failure + def test_against_proc_stat(self): + with open("/proc/stat") as f: + line = f.readline() + ticks = [float(x) for x in line.split()[1:]] + fields = [t / CLOCK_TICKS for t in ticks] + ct = psutil.cpu_times() + TOLERANCE = 1 # 1 second + assert abs(ct.user - fields[0]) < TOLERANCE + assert abs(ct.nice - fields[1]) < TOLERANCE + assert abs(ct.system - fields[2]) < TOLERANCE + assert abs(ct.idle - fields[3]) < TOLERANCE + assert abs(ct.iowait - fields[4]) < TOLERANCE + assert abs(ct.irq - fields[5]) < TOLERANCE + assert abs(ct.softirq - fields[6]) < TOLERANCE + assert abs(ct.steal - fields[7]) < TOLERANCE + + +class TestCpuStats(LinuxTestCase): + + @staticmethod + def assert_close_to_vmstat(vmstat_value, psutil_value): + # Old procps keeps these counters in 32 bits, so it wraps once + # the kernel goes past 2**32 and psutil doesn't. + if psutil_value >= 2**32 and vmstat_value < 2**32: + return pytest.skip("vmstat truncated the counter to 32 bits") + assert abs(vmstat_value - psutil_value) < 500 + + @isolated + def test_ctx_switches(self): + vmstat_value = vmstat("context switches") + psutil_value = psutil.cpu_stats().ctx_switches + self.assert_close_to_vmstat(vmstat_value, psutil_value) + + @isolated + def test_interrupts(self): + vmstat_value = vmstat("interrupts") + psutil_value = psutil.cpu_stats().interrupts + self.assert_close_to_vmstat(vmstat_value, psutil_value) + + +class TestLoadAvg(LinuxTestCase): + def test_getloadavg(self): + psutil_value = psutil.getloadavg() + with open("/proc/loadavg") as f: + proc_value = f.read().split() + + assert abs(float(proc_value[0]) - psutil_value[0]) < 1 + assert abs(float(proc_value[1]) - psutil_value[1]) < 1 + assert abs(float(proc_value[2]) - psutil_value[2]) < 1 + + +# ===================================================================== +# --- system network +# ===================================================================== + + +class TestNetIfAddrs(LinuxTestCase): + def test_ips(self): + for name, addrs in psutil.net_if_addrs().items(): + for addr in addrs: + if addr.family == psutil.AF_LINK: + assert addr.address == get_mac_address(name) + elif addr.family == socket.AF_INET: + assert addr.address == get_ipv4_address(name) + assert addr.netmask == get_ipv4_netmask(name) + if addr.broadcast is not None: + assert addr.broadcast == get_ipv4_broadcast(name) + else: + # SIOCGIFBRDADDR shares a union with the peer + # address, so on a /32 it echoes the address + # back and on a ptp link it gives the peer. + assert get_ipv4_broadcast(name) in { + '0.0.0.0', + addr.address, + addr.ptp, + } + elif addr.family == socket.AF_INET6: + # IPv6 addresses can have a percent symbol at the end. + # E.g. these 2 are equivalent: + # "fe80::1ff:fe23:4567:890a" + # "fe80::1ff:fe23:4567:890a%eth0" + # That is the "zone id" portion, which usually is the name + # of the network interface. + address = addr.address.split('%')[0] + assert address in get_ipv6_addresses(name) + + @requires_cli("ip") + @retry_on_failure + def test_against_ip_addr_v4(self): + # Parse IPv4 addresses per interface from `ip addr` output and + # compare against psutil. Use the label at the end of each inet + # line as the interface name, since it reflects aliases like + # "vboxnet0:avahi" that psutil also uses as keys. + out = sh("ip addr") + ip_addrs = {} # {ifname: [addr, ...]} + for line in out.splitlines(): + # " inet 1.2.3.4/24 brd ... scope global eth0" + m = re.match(r'^\s+inet\s+(\S+).*\s+(\S+)$', line) + if m: + addr = m.group(1).split('/')[0] + ifname = m.group(2) + ip_addrs.setdefault(ifname, []).append(addr) + psutil_addrs = psutil.net_if_addrs() + for ifname, addrs in ip_addrs.items(): + if ifname not in psutil_addrs: + continue + psutil_ipv4 = { + a.address + for a in psutil_addrs[ifname] + if a.family == socket.AF_INET + } + for addr in addrs: + assert addr in psutil_ipv4 + + @requires_cli("ip") + @retry_on_failure + def test_against_ip_addr_v6(self): + # Parse IPv6 addresses per interface from `ip addr` output and + # compare against psutil. Unlike inet, inet6 lines have no label, + # so the interface name comes from the header line. + out = sh("ip addr") + ip_addrs = {} # {ifname: [addr, ...]} + current_if = None + for line in out.splitlines(): + m = re.match(r'^\d+:\s+(\S+):', line) + if m: + current_if = m.group(1).rstrip(':') + m = re.match(r'^\s+inet6\s+(\S+)', line) + if m and current_if: + addr = m.group(1).split('/')[0] + ip_addrs.setdefault(current_if, []).append(addr) + psutil_addrs = psutil.net_if_addrs() + for ifname, addrs in ip_addrs.items(): + if ifname not in psutil_addrs: + continue + # psutil may append %ifname zone ID to link-local addresses. + psutil_ipv6 = { + a.address.split('%')[0] + for a in psutil_addrs[ifname] + if a.family == socket.AF_INET6 + } + for addr in addrs: + assert addr in psutil_ipv6 + + @requires_cli("ip") + def test_net_if_names(self): + out = sh("ip addr").strip() + nics = [x for x in psutil.net_if_addrs() if ':' not in x] + found = 0 + for line in out.split('\n'): + line = line.strip() + if re.search(r"^\d+:", line): + found += 1 + name = line.split(':')[1].strip().split('@')[0] + assert name in nics + assert len(nics) == found + + +class TestNetIfStats(LinuxTestCase): + @requires_cli("ifconfig") + def test_against_ifconfig(self): + for name, stats in psutil.net_if_stats().items(): + try: + out = sh(f"ifconfig {name}") + except RuntimeError: + pass + else: + assert stats.isup == ('RUNNING' in out), out + assert stats.mtu == int( + re.findall(r'(?i)MTU[: ](\d+)', out)[0] + ) + + def test_mtu(self): + for name, stats in psutil.net_if_stats().items(): + with open(f"/sys/class/net/{name}/mtu") as f: + assert stats.mtu == int(f.read().strip()) + + @requires_cli("ifconfig") + def test_flags(self): + # first line looks like this: + # "eth0: flags=4163 mtu 1500" + matches_found = 0 + for name, stats in psutil.net_if_stats().items(): + try: + out = sh(f"ifconfig {name}") + except RuntimeError: + pass + else: + match = re.search(r"flags=(\d+)?<(.*?)>", out) + if match and len(match.groups()) >= 2: + matches_found += 1 + ifconfig_flags = set(match.group(2).lower().split(",")) + psutil_flags = set(stats.flags.split(",")) + assert ifconfig_flags == psutil_flags + else: + # ifconfig has a different output on CentOS 6 + # let's try that + match = re.search(r"(.*) MTU:(\d+) Metric:(\d+)", out) + if match and len(match.groups()) >= 3: + matches_found += 1 + ifconfig_flags = set(match.group(1).lower().split()) + psutil_flags = set(stats.flags.split(",")) + assert ifconfig_flags == psutil_flags + + if not matches_found: + return pytest.fail("no matches were found") + + +class TestNetIoCounters(LinuxTestCase): + @requires_cli("ifconfig") + @retry_on_failure + def test_against_ifconfig(self): + def ifconfig(nic): + ret = {} + out = sh(f"ifconfig {nic}") + ret['packets_recv'] = int( + re.findall(r'RX packets[: ](\d+)', out)[0] + ) + ret['packets_sent'] = int( + re.findall(r'TX packets[: ](\d+)', out)[0] + ) + ret['errin'] = int(re.findall(r'errors[: ](\d+)', out)[0]) + ret['errout'] = int(re.findall(r'errors[: ](\d+)', out)[1]) + ret['dropin'] = int(re.findall(r'dropped[: ](\d+)', out)[0]) + ret['dropout'] = int(re.findall(r'dropped[: ](\d+)', out)[1]) + ret['bytes_recv'] = int( + re.findall(r'RX (?:packets \d+ +)?bytes[: ](\d+)', out)[0] + ) + ret['bytes_sent'] = int( + re.findall(r'TX (?:packets \d+ +)?bytes[: ](\d+)', out)[0] + ) + return ret + + nio = psutil.net_io_counters(pernic=True, nowrap=False) + for name, stats in nio.items(): + try: + ifconfig_ret = ifconfig(name) + except RuntimeError: + continue + if not any(ifconfig_ret.values()): + # net-tools can't parse /proc/net/dev lines whose NIC + # name fills the whole 15 chars (e.g. enxbaa44ee7dd5e), + # and prints zeros for the whole interface. + continue + + assert ( + abs(stats.bytes_recv - ifconfig_ret['bytes_recv']) < 1024 * 10 + ) + assert ( + abs(stats.bytes_sent - ifconfig_ret['bytes_sent']) < 1024 * 10 + ) + assert ( + abs(stats.packets_recv - ifconfig_ret['packets_recv']) < 1024 + ) + assert ( + abs(stats.packets_sent - ifconfig_ret['packets_sent']) < 1024 + ) + assert abs(stats.errin - ifconfig_ret['errin']) < 10 + assert abs(stats.errout - ifconfig_ret['errout']) < 10 + assert abs(stats.dropin - ifconfig_ret['dropin']) < 10 + assert abs(stats.dropout - ifconfig_ret['dropout']) < 10 + + +class TestNetConnections(LinuxTestCase): + @mock.patch('psutil._pslinux.socket.inet_ntop', side_effect=ValueError) + @mock.patch('psutil._pslinux.supports_ipv6', return_value=False) + def test_emulate_ipv6_unsupported(self, supports_ipv6, inet_ntop): + # see: https://github.com/giampaolo/psutil/issues/623 + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + try: + s.bind(("::1", 0)) + except OSError: + pass + psutil.net_connections(kind='inet6') + + def test_emulate_unix(self): + content = textwrap.dedent("""\ + 0: 00000003 000 000 0001 03 462170 @/tmp/dbus-Qw2hMPIU3n + 0: 00000003 000 000 0001 03 35010 @/tmp/dbus-tB2X8h69BQ + 0: 00000003 000 000 0001 03 34424 @/tmp/dbus-cHy80Y8O + 000000000000000000000000000000000000000000000000000000 + """) + with mock_open_content({"/proc/net/unix": content}) as m: + psutil.net_connections(kind='unix') + assert m.called + + @serial + @requires_cli("ss") + @retry_on_failure + def test_against_ss(self): + # Listening ports are more stable, so an exact set comparison is + # more reliable. + out = sh(["ss", "-tuanp"]) + ss_ports = set() + for line in out.splitlines(): + fields = line.split() + if ( + len(fields) >= 5 + and fields[0] == "tcp" + and fields[1] == "LISTEN" + ): + port = int(fields[4].rsplit(":", 1)[-1]) + ss_ports.add(port) + psutil_ports = { + c.laddr.port + for c in psutil.net_connections(kind="tcp") + if c.status == psutil.CONN_LISTEN + } + assert ss_ports == psutil_ports + + +# ===================================================================== +# --- system disks +# ===================================================================== + + +class TestDiskPartitions(LinuxTestCase): + @skipif(not hasattr(os, 'statvfs'), reason="os.statvfs() not available") + @skip_on_not_implemented + def test_against_df(self): + # test psutil.disk_usage() and psutil.disk_partitions() + # against "df -a" + def df(path): + out = sh(f'df -P -B 1 "{path}"').strip() + lines = out.split('\n') + lines.pop(0) + line = lines.pop(0) + dev, total, used, free = line.split()[:4] + if dev == 'none': + dev = '' + total, used, free = int(total), int(used), int(free) + return dev, total, used, free + + for part in psutil.disk_partitions(all=False): + usage = psutil.disk_usage(part.mountpoint) + _, total, used, free = df(part.mountpoint) + assert usage.total == total + assert abs(usage.free - free) < TOLERANCE_DISK_USAGE + assert abs(usage.used - used) < TOLERANCE_DISK_USAGE + + def test_zfs_fs(self): + # Test that ZFS partitions are returned. + with open("/proc/filesystems") as f: + data = f.read() + if 'zfs' in data: + for part in psutil.disk_partitions(): + if part.fstype == 'zfs': + return + + # No ZFS partitions on this system. Let's fake one. + fake_file = io.StringIO("nodev\tzfs\n") + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m1: + with mock.patch.object( + _psutil, + 'disk_partitions', + return_value=[('/dev/sdb3', '/', 'zfs', 'rw')], + ) as m2: + ret = psutil.disk_partitions() + assert m1.called + assert m2.called + assert ret + assert ret[0].fstype == 'zfs' + + def test_emulate_realpath_fail(self): + # See: https://github.com/giampaolo/psutil/issues/1307 + try: + with mock.patch( + 'os.path.realpath', return_value='/non/existent' + ) as m: + with pytest.raises(FileNotFoundError): + psutil.disk_partitions() + assert m.called + finally: + psutil.PROCFS_PATH = "/proc" + + +class TestDiskIoCounters(LinuxTestCase): + def test_emulate_kernel_2_4(self): + # Tests /proc/diskstats parsing format for 2.4 kernels, see: + # https://github.com/giampaolo/psutil/issues/767 + content = " 3 0 1 hda 2 3 4 5 6 7 8 9 10 11 12" + with mock_open_content({'/proc/diskstats': content}): + with mock.patch( + 'psutil._pslinux.is_storage_device', return_value=True + ): + ret = psutil.disk_io_counters(nowrap=False) + assert ret.read_count == 1 + assert ret.read_merged_count == 2 + assert ret.read_bytes == 3 * SECTOR_SIZE + assert ret.read_time == 4 + assert ret.write_count == 5 + assert ret.write_merged_count == 6 + assert ret.write_bytes == 7 * SECTOR_SIZE + assert ret.write_time == 8 + assert ret.busy_time == 10 + + def test_emulate_kernel_2_6_full(self): + # Tests /proc/diskstats parsing format for 2.6 kernels, + # lines reporting all metrics: + # https://github.com/giampaolo/psutil/issues/767 + content = " 3 0 hda 1 2 3 4 5 6 7 8 9 10 11" + with mock_open_content({"/proc/diskstats": content}): + with mock.patch( + 'psutil._pslinux.is_storage_device', return_value=True + ): + ret = psutil.disk_io_counters(nowrap=False) + assert ret.read_count == 1 + assert ret.read_merged_count == 2 + assert ret.read_bytes == 3 * SECTOR_SIZE + assert ret.read_time == 4 + assert ret.write_count == 5 + assert ret.write_merged_count == 6 + assert ret.write_bytes == 7 * SECTOR_SIZE + assert ret.write_time == 8 + assert ret.busy_time == 10 + + def test_emulate_kernel_2_6_limited(self): + # Tests /proc/diskstats parsing format for 2.6 kernels, + # where one line of /proc/partitions return a limited + # amount of metrics when it bumps into a partition + # (instead of a disk). See: + # https://github.com/giampaolo/psutil/issues/767 + with mock_open_content({"/proc/diskstats": " 3 1 hda 1 2 3 4"}): + with mock.patch( + 'psutil._pslinux.is_storage_device', return_value=True + ): + ret = psutil.disk_io_counters(nowrap=False) + assert ret.read_count == 1 + assert ret.read_bytes == 2 * SECTOR_SIZE + assert ret.write_count == 3 + assert ret.write_bytes == 4 * SECTOR_SIZE + + assert ret.read_merged_count == 0 + assert ret.read_time == 0 + assert ret.write_merged_count == 0 + assert ret.write_time == 0 + assert ret.busy_time == 0 + + def test_emulate_include_partitions(self): + # Make sure that when perdisk=True disk partitions are returned, + # see: + # https://github.com/giampaolo/psutil/pull/1313#issuecomment-408626842 + content = textwrap.dedent("""\ + 3 0 nvme0n1 1 2 3 4 5 6 7 8 9 10 11 + 3 0 nvme0n1p1 1 2 3 4 5 6 7 8 9 10 11 + """) + with mock_open_content({"/proc/diskstats": content}): + with mock.patch( + 'psutil._pslinux.is_storage_device', return_value=False + ): + ret = psutil.disk_io_counters(perdisk=True, nowrap=False) + assert len(ret) == 2 + assert ret['nvme0n1'].read_count == 1 + assert ret['nvme0n1p1'].read_count == 1 + assert ret['nvme0n1'].write_count == 5 + assert ret['nvme0n1p1'].write_count == 5 + + def test_emulate_exclude_partitions(self): + # Make sure that when perdisk=False partitions (e.g. 'sda1', + # 'nvme0n1p1') are skipped and not included in the total count. + # https://github.com/giampaolo/psutil/pull/1313#issuecomment-408626842 + content = textwrap.dedent("""\ + 3 0 nvme0n1 1 2 3 4 5 6 7 8 9 10 11 + 3 0 nvme0n1p1 1 2 3 4 5 6 7 8 9 10 11 + """) + with mock_open_content({"/proc/diskstats": content}): + with mock.patch( + 'psutil._pslinux.is_storage_device', return_value=False + ): + ret = psutil.disk_io_counters(perdisk=False, nowrap=False) + assert ret is None + + def is_storage_device(name): + return name == 'nvme0n1' + + content = textwrap.dedent("""\ + 3 0 nvme0n1 1 2 3 4 5 6 7 8 9 10 11 + 3 0 nvme0n1p1 1 2 3 4 5 6 7 8 9 10 11 + """) + with mock_open_content({"/proc/diskstats": content}): + with mock.patch( + 'psutil._pslinux.is_storage_device', + create=True, + side_effect=is_storage_device, + ): + ret = psutil.disk_io_counters(perdisk=False, nowrap=False) + assert ret.read_count == 1 + assert ret.write_count == 5 + + def test_emulate_use_sysfs(self): + def exists(path): + return path == '/proc/diskstats' + + wprocfs = psutil.disk_io_counters(perdisk=True) + with mock.patch( + 'psutil._pslinux.os.path.exists', create=True, side_effect=exists + ): + wsysfs = psutil.disk_io_counters(perdisk=True) + assert len(wprocfs) == len(wsysfs) + + def test_emulate_not_impl(self): + def exists(path): + return False + + with mock.patch( + 'psutil._pslinux.os.path.exists', create=True, side_effect=exists + ): + with pytest.raises(NotImplementedError): + psutil.disk_io_counters() + + @requires_cli("iostat") + @retry_on_failure + def test_against_iostat(self): + # Cross-check read_bytes/write_bytes against 'iostat -d -k' + # cumulative totals (kB_read, kB_wrtn columns). + out = sh(["iostat", "-d", "-k"]) + iostat_disks = {} + for line in out.splitlines(): + fields = line.split() + if len(fields) < 7 or fields[0] in {"Linux", "Device"}: + continue + name = fields[0] + try: + kb_read = int(fields[5]) + kb_wrtn = int(fields[6]) + except ValueError: + continue + iostat_disks[name] = (kb_read * 1024, kb_wrtn * 1024) + + psutil_disks = psutil.disk_io_counters(perdisk=True, nowrap=False) + for name, (bytes_read, bytes_wrtn) in iostat_disks.items(): + if name not in psutil_disks: + continue + stats = psutil_disks[name] + assert abs(stats.read_bytes - bytes_read) < 1024 * 1024 + assert abs(stats.write_bytes - bytes_wrtn) < 1024 * 1024 + + +class TestRootFsDeviceFinder(LinuxTestCase): + def setUp(self): + dev = os.stat("/").st_dev + self.major = os.major(dev) + self.minor = os.minor(dev) + + def test_call_methods(self): + finder = RootFsDeviceFinder() + if os.path.exists("/proc/partitions"): + finder.ask_proc_partitions() + else: + with pytest.raises(FileNotFoundError): + finder.ask_proc_partitions() + if os.path.exists(f"/sys/dev/block/{self.major}:{self.minor}/uevent"): + finder.ask_sys_dev_block() + else: + with pytest.raises(FileNotFoundError): + finder.ask_sys_dev_block() + finder.ask_sys_class_block() + + @skipif(not ROOTFS_ON_BLOCK_DEV, reason="/ is not on a block device") + def test_comparisons(self): + finder = RootFsDeviceFinder() + assert finder.find() is not None + + a = b = c = None + if os.path.exists("/proc/partitions"): + a = finder.ask_proc_partitions() + if os.path.exists(f"/sys/dev/block/{self.major}:{self.minor}/uevent"): + b = finder.ask_sys_class_block() + c = finder.ask_sys_dev_block() + + base = a or b or c + if base and a: + assert base == a + if base and b: + assert base == b + if base and c: + assert base == c + + @requires_cli("findmnt") + @skipif(not ROOTFS_ON_BLOCK_DEV, reason="/ is not on a block device") + def test_against_findmnt(self): + psutil_value = RootFsDeviceFinder().find() + findmnt_value = sh("findmnt -o SOURCE -rn /") + # findmnt prints the friendly alias (e.g. /dev/mapper/vg-root), + # psutil the kernel name it points to (e.g. /dev/dm-0). + assert os.path.realpath(psutil_value) == os.path.realpath( + findmnt_value + ) + + def test_disk_partitions_mocked(self): + with mock.patch.object( + _psutil, + 'disk_partitions', + return_value=[('/dev/root', '/', 'ext4', 'rw')], + ) as m: + part = psutil.disk_partitions(all=True)[0] + assert m.called + assert part.device != "/dev/root" + assert part.device == RootFsDeviceFinder().find() + + +# ===================================================================== +# --- misc +# ===================================================================== + + +class TestMisc(LinuxTestCase): + def test_boot_time(self): + vmstat_value = vmstat('boot time') + psutil_value = psutil.boot_time() + assert int(vmstat_value) == int(psutil_value) + + def test_no_procfs_on_import(self): + my_procfs = self.get_testfn() + os.mkdir(my_procfs) + + with open(os.path.join(my_procfs, 'stat'), 'w') as f: + f.write('cpu 0 0 0 0 0 0 0 0 0 0\n') + f.write('cpu0 0 0 0 0 0 0 0 0 0 0\n') + f.write('cpu1 0 0 0 0 0 0 0 0 0 0\n') + + try: + orig_open = open + + def open_mock(name, *args, **kwargs): + if name.startswith('/proc'): + raise FileNotFoundError + return orig_open(name, *args, **kwargs) + + with mock.patch("builtins.open", side_effect=open_mock): + reload_module(psutil) + + with pytest.raises(OSError): + psutil.cpu_times() + with pytest.raises(OSError): + psutil.cpu_times(percpu=True) + with pytest.raises(OSError): + psutil.cpu_percent() + with pytest.raises(OSError): + psutil.cpu_percent(percpu=True) + with pytest.raises(OSError): + psutil.cpu_times_percent() + with pytest.raises(OSError): + psutil.cpu_times_percent(percpu=True) + + psutil.PROCFS_PATH = my_procfs + + assert psutil.cpu_percent() == 0 + assert sum(psutil.cpu_times_percent()) == 0 + + # since we don't know the number of CPUs at import time, + # we awkwardly say there are none until the second call + per_cpu_percent = psutil.cpu_percent(percpu=True) + assert sum(per_cpu_percent) == 0 + + # ditto awkward length + per_cpu_times_percent = psutil.cpu_times_percent(percpu=True) + assert sum(map(sum, per_cpu_times_percent)) == 0 + + # much user, very busy + with open(os.path.join(my_procfs, 'stat'), 'w') as f: + f.write('cpu 1 0 0 0 0 0 0 0 0 0\n') + f.write('cpu0 1 0 0 0 0 0 0 0 0 0\n') + f.write('cpu1 1 0 0 0 0 0 0 0 0 0\n') + + assert psutil.cpu_percent() != 0 + assert sum(psutil.cpu_percent(percpu=True)) != 0 + assert sum(psutil.cpu_times_percent()) != 0 + assert ( + sum(map(sum, psutil.cpu_times_percent(percpu=True))) != 0 + ) + finally: + shutil.rmtree(my_procfs) + reload_module(psutil) + + assert psutil.PROCFS_PATH == '/proc' + + def test_cpu_steal_decrease(self): + # Test cumulative cpu stats decrease. We should ignore this. + # See issue #1210. + content = textwrap.dedent("""\ + cpu 0 0 0 0 0 0 0 1 0 0 + cpu0 0 0 0 0 0 0 0 1 0 0 + cpu1 0 0 0 0 0 0 0 1 0 0 + """).encode() + with mock_open_content({"/proc/stat": content}) as m: + # first call to "percent" functions should read the new stat file + # and compare to the "real" file read at import time - so the + # values are meaningless + psutil.cpu_percent() + assert m.called + psutil.cpu_percent(percpu=True) + psutil.cpu_times_percent() + psutil.cpu_times_percent(percpu=True) + + content = textwrap.dedent("""\ + cpu 1 0 0 0 0 0 0 0 0 0 + cpu0 1 0 0 0 0 0 0 0 0 0 + cpu1 1 0 0 0 0 0 0 0 0 0 + """).encode() + with mock_open_content({"/proc/stat": content}): + # Increase "user" while steal goes "backwards" to zero. + cpu_percent = psutil.cpu_percent() + assert m.called + cpu_percent_percpu = psutil.cpu_percent(percpu=True) + cpu_times_percent = psutil.cpu_times_percent() + cpu_times_percent_percpu = psutil.cpu_times_percent(percpu=True) + assert cpu_percent != 0 + assert sum(cpu_percent_percpu) != 0 + assert sum(cpu_times_percent) != 0 + assert sum(cpu_times_percent) != 100.0 + assert sum(map(sum, cpu_times_percent_percpu)) != 0 + assert sum(map(sum, cpu_times_percent_percpu)) != 100.0 + assert cpu_times_percent.steal == 0 + assert cpu_times_percent.user != 0 + + def test_boot_time_mocked(self): + with mock.patch('psutil._common.open', create=True) as m: + with pytest.raises(RuntimeError): + psutil._pslinux.boot_time() + assert m.called + + def test_users(self): + # Make sure the C extension converts ':0' and ':0.0' to + # 'localhost'. + for user in psutil.users(): + assert user.host not in {":0", ":0.0"} + + def test_procfs_path(self): + tdir = self.get_testfn() + os.mkdir(tdir) + try: + psutil.PROCFS_PATH = tdir + with pytest.raises(OSError): + psutil.virtual_memory() + with pytest.raises(OSError): + psutil.cpu_times() + with pytest.raises(OSError): + psutil.cpu_times(percpu=True) + with pytest.raises(OSError): + psutil.boot_time() + with pytest.raises(OSError): + psutil.net_connections() + with pytest.raises(OSError): + psutil.net_io_counters() + with pytest.raises(OSError): + psutil.net_if_stats() + with pytest.raises(OSError): + psutil.disk_partitions() + with pytest.raises(psutil.NoSuchProcess): + psutil.Process() + finally: + psutil.PROCFS_PATH = "/proc" + + @retry_on_failure + @isolated + def test_issue_687(self): + # In case of thread ID: + # - pid_exists() is supposed to return False + # - Process(tid) is supposed to work + # - pids() should not return the TID + # See: https://github.com/giampaolo/psutil/issues/687 + + p = psutil.Process() + nthreads = len(p.threads()) + with ThreadTask(): + threads = p.threads() + assert len(threads) == nthreads + 1 + tid = sorted(threads, key=lambda x: x.id)[1].id + assert p.pid != tid + pt = psutil.Process(tid) + pt.as_dict() + assert tid not in psutil.pids() + + def test_pid_exists_no_proc_status(self): + # Internally pid_exists relies on /proc/{pid}/status. + # Emulate a case where this file is empty in which case + # psutil is supposed to fall back on using pids(). + with mock_open_content({"/proc/%s/status": ""}) as m: + assert psutil.pid_exists(os.getpid()) + assert m.called + + +# ===================================================================== +# --- sensors +# ===================================================================== + + +@skipif(not HAS_BATTERY, reason="no battery") +class TestSensorsBattery(LinuxTestCase): + @requires_cli("acpi") + def test_percent(self): + out = sh("acpi -b") + first_battery = out.split("\n")[0] + acpi_value = int(first_battery.split(",")[1].strip().replace('%', '')) + psutil_value = psutil.sensors_battery().percent + assert abs(acpi_value - psutil_value) < 1 + + def test_emulate_power_plugged(self): + # Pretend the AC power cable is connected. + def open_mock(name, *args, **kwargs): + if name.endswith(('AC0/online', 'AC/online')): + return io.BytesIO(b"1") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock) as m: + assert psutil.sensors_battery().power_plugged is True + assert ( + psutil.sensors_battery().secsleft + == psutil.POWER_TIME_UNLIMITED + ) + assert m.called + + def test_emulate_power_plugged_2(self): + # Same as above but pretend /AC0/online does not exist in which + # case code relies on /status file. + def open_mock(name, *args, **kwargs): + if name.endswith(('AC0/online', 'AC/online')): + raise FileNotFoundError + if name.endswith("/status"): + return io.StringIO("charging") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock) as m: + assert psutil.sensors_battery().power_plugged is True + assert m.called + + def test_emulate_power_not_plugged(self): + # Pretend the AC power cable is not connected. + def open_mock(name, *args, **kwargs): + if name.endswith(('AC0/online', 'AC/online')): + return io.BytesIO(b"0") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock) as m: + assert psutil.sensors_battery().power_plugged is False + assert m.called + + def test_emulate_power_not_plugged_2(self): + # Same as above but pretend /AC0/online does not exist in which + # case code relies on /status file. + def open_mock(name, *args, **kwargs): + if name.endswith(('AC0/online', 'AC/online')): + raise FileNotFoundError + if name.endswith("/status"): + return io.StringIO("discharging") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock) as m: + assert psutil.sensors_battery().power_plugged is False + assert m.called + + def test_emulate_power_undetermined(self): + # Pretend we can't know whether the AC power cable not + # connected (assert fallback to False). + def open_mock(name, *args, **kwargs): + if name.startswith(( + '/sys/class/power_supply/AC0/online', + '/sys/class/power_supply/AC/online', + )): + raise FileNotFoundError + if name.startswith("/sys/class/power_supply/BAT0/status"): + return io.BytesIO(b"???") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock) as m: + assert psutil.sensors_battery().power_plugged is None + assert m.called + + def test_emulate_energy_full_0(self): + # Emulate a case where energy_full files returns 0. + with mock_open_content( + {"/sys/class/power_supply/BAT0/energy_full": b"0"} + ) as m: + assert psutil.sensors_battery().percent == 0 + assert m.called + + def test_emulate_energy_full_not_avail(self): + # Emulate a case where energy_full file does not exist. + # Expected fallback on /capacity. + with mock_open_exception( + "/sys/class/power_supply/BAT0/energy_full", + FileNotFoundError, + ): + with mock_open_exception( + "/sys/class/power_supply/BAT0/charge_full", + FileNotFoundError, + ): + with mock_open_content( + {"/sys/class/power_supply/BAT0/capacity": b"88"} + ): + assert psutil.sensors_battery().percent == 88 + + @skipif( + not os.path.isfile("/sys/class/power_supply/BAT0/capacity"), + reason="BAT /capacity file don't exist", + ) + def test_percent_against_capacity(self): + # Internally, if we have /energy_full, the percentage will be + # calculated by NOT reading the /capacity file, to get more + # accuracy. Check against /capacity to make sure our percentage + # is calculated correctly. + with open("/sys/class/power_supply/BAT0/capacity") as f: + capacity = float(f.read()) + assert psutil.sensors_battery().percent == pytest.approx( + capacity, abs=1 + ) + + def test_emulate_no_power(self): + # Emulate a case where /AC0/online file nor /BAT0/status exist. + with mock_open_exception( + "/sys/class/power_supply/AC/online", FileNotFoundError + ): + with mock_open_exception( + "/sys/class/power_supply/AC0/online", FileNotFoundError + ): + with mock_open_exception( + "/sys/class/power_supply/BAT0/status", + FileNotFoundError, + ): + assert psutil.sensors_battery().power_plugged is None + + def test_fully_emulated(self): + def open_mock(name, *args, **kwargs): + if name.endswith("/energy_now"): + return io.StringIO("60000000") + elif name.endswith("/power_now"): + return io.StringIO("0") + elif name.endswith("/energy_full"): + return io.StringIO("60000001") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch('os.listdir', return_value=["BAT0"]) as mlistdir: + with mock.patch("builtins.open", side_effect=open_mock) as mopen: + assert psutil.sensors_battery() is not None + assert mlistdir.called + assert mopen.called + + +class TestSensorsTemperatures(LinuxTestCase): + def test_emulate_class_hwmon(self): + def open_mock(name, *args, **kwargs): + if name.endswith('/name'): + return io.StringIO("name") + elif name.endswith('/temp1_label'): + return io.StringIO("label") + elif name.endswith('/temp1_input'): + return io.BytesIO(b"30000") + elif name.endswith('/temp1_max'): + return io.BytesIO(b"40000") + elif name.endswith('/temp1_crit'): + return io.BytesIO(b"50000") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock): + # Test case with /sys/class/hwmon + with mock.patch( + 'glob.glob', return_value=['/sys/class/hwmon/hwmon0/temp1'] + ): + temp = psutil.sensors_temperatures()['name'][0] + assert temp.label == 'label' + assert temp.current == 30.0 + assert temp.high == 40.0 + assert temp.critical == 50.0 + + def test_emulate_class_thermal(self): + def open_mock(name, *args, **kwargs): + if name.endswith('0_temp'): + return io.BytesIO(b"50000") + elif name.endswith('temp'): + return io.BytesIO(b"30000") + elif name.endswith('0_type'): + return io.StringIO("critical") + elif name.endswith('type'): + return io.StringIO("name") + else: + return orig_open(name, *args, **kwargs) + + def glob_mock(path): + if path in { + '/sys/class/hwmon/hwmon*/temp*_*', + '/sys/class/hwmon/hwmon*/device/temp*_*', + }: + return [] + elif path == '/sys/class/thermal/thermal_zone*': + return ['/sys/class/thermal/thermal_zone0'] + elif path == '/sys/class/thermal/thermal_zone0/trip_point*': + return [ + '/sys/class/thermal/thermal_zone1/trip_point_0_type', + '/sys/class/thermal/thermal_zone1/trip_point_0_temp', + ] + return [] + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock): + with mock.patch('glob.glob', create=True, side_effect=glob_mock): + temp = psutil.sensors_temperatures()['name'][0] + assert temp.label == '' + assert temp.current == 30.0 + assert temp.high == 50.0 + assert temp.critical == 50.0 + + +class TestSensorsFans(LinuxTestCase): + def test_emulate_data(self): + def open_mock(name, *args, **kwargs): + if name.endswith('/name'): + return io.StringIO("name") + elif name.endswith('/fan1_label'): + return io.StringIO("label") + elif name.endswith('/fan1_input'): + return io.StringIO("2000") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock): + with mock.patch( + 'glob.glob', return_value=['/sys/class/hwmon/hwmon2/fan1'] + ): + fan = psutil.sensors_fans()['name'][0] + assert fan.label == 'label' + assert fan.current == 2000 + + +# ===================================================================== +# --- test process +# ===================================================================== + + +class TestProcess(LinuxTestCase): + @retry_on_failure + def test_parse_smaps_vs_memory_maps(self): + sproc = self.spawn_subproc() + uss, pss, swap = psutil._pslinux.Process(sproc.pid)._parse_smaps() + maps = psutil.Process(sproc.pid).memory_maps(grouped=False) + assert ( + abs(uss - sum(x.private_dirty + x.private_clean for x in maps)) + < 4096 + ) + assert abs(pss - sum(x.pss for x in maps)) < 4096 + assert abs(swap - sum(x.swap for x in maps)) < 4096 + + def test_parse_smaps_mocked(self): + # See: https://github.com/giampaolo/psutil/issues/1222 + content = textwrap.dedent("""\ + fffff0 r-xp 00000000 00:00 0 [vsyscall] + Size: 1 kB + Rss: 2 kB + Pss: 3 kB + Shared_Clean: 4 kB + Shared_Dirty: 5 kB + Private_Clean: 6 kB + Private_Dirty: 7 kB + Referenced: 8 kB + Anonymous: 9 kB + LazyFree: 10 kB + AnonHugePages: 11 kB + ShmemPmdMapped: 12 kB + Shared_Hugetlb: 13 kB + Private_Hugetlb: 14 kB + Swap: 15 kB + SwapPss: 16 kB + KernelPageSize: 17 kB + MMUPageSize: 18 kB + Locked: 19 kB + VmFlags: rd ex + """).encode() + with mock_open_content({f"/proc/{os.getpid()}/smaps": content}) as m: + p = psutil._pslinux.Process(os.getpid()) + uss, pss, swap = p._parse_smaps() + assert m.called + assert uss == (6 + 7 + 14) * 1024 + assert pss == 3 * 1024 + assert swap == 15 * 1024 + + def test_open_files_mode(self): + def get_test_file(fname): + p = psutil.Process() + giveup_at = time.monotonic() + GLOBAL_TIMEOUT + while True: + for file in p.open_files(): + if file.path == os.path.abspath(fname): + return file + elif time.monotonic() > giveup_at: + break + raise RuntimeError("timeout looking for test file") + + testfn = self.get_testfn() + with open(testfn, "w"): + assert get_test_file(testfn).mode == "w" + with open(testfn): + assert get_test_file(testfn).mode == "r" + with open(testfn, "a"): + assert get_test_file(testfn).mode == "a" + with open(testfn, "r+"): + assert get_test_file(testfn).mode == "r+" + with open(testfn, "w+"): + assert get_test_file(testfn).mode == "r+" + with open(testfn, "a+"): + assert get_test_file(testfn).mode == "a+" + + safe_rmpath(testfn) + with open(testfn, "x"): + assert get_test_file(testfn).mode == "w" + safe_rmpath(testfn) + with open(testfn, "x+"): + assert get_test_file(testfn).mode == "r+" + + def test_open_files_file_gone(self): + # simulates a file which gets deleted during open_files() + # execution + p = psutil.Process() + files = p.open_files() + with open(self.get_testfn(), 'w'): + # give the kernel some time to see the new file + call_until(lambda: len(p.open_files()) != len(files)) + with mock.patch( + 'psutil._pslinux.os.readlink', + side_effect=FileNotFoundError, + ) as m: + assert p.open_files() == [] + assert m.called + # also simulate the case where os.readlink() returns EINVAL + # in which case psutil is supposed to 'continue' + with mock.patch( + 'psutil._pslinux.os.readlink', + side_effect=OSError(errno.EINVAL, ""), + ) as m: + assert p.open_files() == [] + assert m.called + + def test_open_files_fd_gone(self): + # Simulate a case where /proc/{pid}/fdinfo/{fd} disappears + # while iterating through fds. + # https://travis-ci.org/giampaolo/psutil/jobs/225694530 + p = psutil.Process() + files = p.open_files() + with open(self.get_testfn(), 'w'): + # give the kernel some time to see the new file + call_until(lambda: len(p.open_files()) != len(files)) + with mock.patch( + "builtins.open", side_effect=FileNotFoundError + ) as m: + assert p.open_files() == [] + assert m.called + + def test_open_files_enametoolong(self): + # Simulate a case where /proc/{pid}/fd/{fd} symlink + # points to a file with full path longer than PATH_MAX, see: + # https://github.com/giampaolo/psutil/issues/1940 + p = psutil.Process() + files = p.open_files() + with open(self.get_testfn(), 'w'): + # give the kernel some time to see the new file + call_until(lambda: len(p.open_files()) != len(files)) + patch_point = 'psutil._pslinux.os.readlink' + with mock.patch( + patch_point, side_effect=OSError(errno.ENAMETOOLONG, "") + ) as m: + with mock.patch("psutil._pslinux.debug"): + assert p.open_files() == [] + assert m.called + + # --- mocked tests + + def test_terminal_mocked(self): + with mock.patch( + 'psutil._pslinux._psposix._get_terminal_map', return_value={} + ): + assert psutil._pslinux.Process(os.getpid()).terminal() is None + + def test_cmdline_mocked(self): + # see: https://github.com/giampaolo/psutil/issues/639 + p = psutil.Process() + fake_file = io.StringIO('foo\x00bar\x00') + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m: + assert p.cmdline() == ['foo', 'bar'] + assert m.called + fake_file = io.StringIO('foo\x00bar\x00\x00') + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m: + assert p.cmdline() == ['foo', 'bar', ''] + assert m.called + + def test_cmdline_spaces_mocked(self): + # see: https://github.com/giampaolo/psutil/issues/1179 + p = psutil.Process() + fake_file = io.StringIO('foo bar ') + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m: + assert p.cmdline() == ['foo', 'bar'] + assert m.called + fake_file = io.StringIO('foo bar ') + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m: + assert p.cmdline() == ['foo', 'bar', ''] + assert m.called + + def test_cmdline_mixed_separators(self): + # https://github.com/giampaolo/psutil/issues/1179#issuecomment-552984549 + p = psutil.Process() + fake_file = io.StringIO('foo\x20bar\x00') + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m: + assert p.cmdline() == ['foo', 'bar'] + assert m.called + + def test_readlink_path_deleted_mocked(self): + with mock.patch( + 'psutil._pslinux.os.readlink', return_value='/home/foo (deleted)' + ): + assert psutil.Process().exe() == "/home/foo" + assert psutil.Process().cwd() == "/home/foo" + + def test_threads_mocked(self): + # Test the case where os.listdir() returns a file (thread) + # which no longer exists by the time we open() it (race + # condition). threads() is supposed to ignore that instead + # of raising NSP. + def open_mock_1(name, *args, **kwargs): + if name.startswith(f"/proc/{os.getpid()}/task"): + raise FileNotFoundError + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock_1) as m: + ret = psutil.Process().threads() + assert m.called + assert ret == [] + + # ...but if it bumps into something != ENOENT we want an + # exception. + def open_mock_2(name, *args, **kwargs): + if name.startswith(f"/proc/{os.getpid()}/task"): + raise PermissionError + return orig_open(name, *args, **kwargs) + + with mock.patch("builtins.open", side_effect=open_mock_2): + with pytest.raises(psutil.AccessDenied): + psutil.Process().threads() + + def test_exe_mocked(self): + with mock.patch( + 'psutil._pslinux.readlink', side_effect=FileNotFoundError + ) as m: + # de-activate guessing from cmdline() + with mock.patch( + 'psutil._pslinux.Process.cmdline', return_value=[] + ): + ret = psutil.Process().exe() + assert m.called + assert ret == "" + + def test_cwd_mocked(self): + # https://github.com/giampaolo/psutil/issues/2514 + with mock.patch( + 'psutil._pslinux.readlink', side_effect=FileNotFoundError + ) as m: + ret = psutil.Process().cwd() + assert m.called + assert ret == "" + + def test_issue_1014(self): + # Emulates a case where smaps file does not exist. In this case + # wrap_exception decorator should not raise NoSuchProcess. + with mock_open_exception( + f"/proc/{os.getpid()}/smaps", FileNotFoundError + ) as m: + p = psutil.Process() + with pytest.raises(FileNotFoundError): + p.memory_maps() + assert m.called + + def test_issue_2418(self): + p = psutil.Process() + with mock_open_exception( + f"/proc/{os.getpid()}/statm", FileNotFoundError + ): + with mock.patch("os.path.exists", return_value=False): + with pytest.raises(psutil.NoSuchProcess): + p.memory_info() + + @skipif(not HAS_PROC_RLIMIT, reason="not supported") + def test_rlimit_zombie(self): + # Emulate a case where rlimit() raises ENOSYS, which may + # happen in case of zombie process: + # https://travis-ci.org/giampaolo/psutil/jobs/51368273 + with mock.patch( + "resource.prlimit", side_effect=OSError(errno.ENOSYS, "") + ) as m1: + with mock.patch( + "psutil._pslinux.Process._is_zombie", return_value=True + ) as m2: + p = psutil.Process() + p.name() + with pytest.raises(psutil.ZombieProcess) as cm: + p.rlimit(psutil.RLIMIT_NOFILE) + assert m1.called + assert m2.called + assert cm.value.pid == p.pid + assert cm.value.name == p.name() + + def test_stat_file_parsing(self): + args = [ + "0", # pid + "(cat)", # name + "Z", # status + "1", # ppid + "0", # pgrp + "0", # session + "0", # tty + "0", # tpgid + "0", # flags + "0", # minflt + "0", # cminflt + "0", # majflt + "0", # cmajflt + "2", # utime + "3", # stime + "4", # cutime + "5", # cstime + "0", # priority + "0", # nice + "0", # num_threads + "0", # itrealvalue + "6", # starttime + "0", # vsize + "0", # rss + "0", # rsslim + "0", # startcode + "0", # endcode + "0", # startstack + "0", # kstkesp + "0", # kstkeip + "0", # signal + "0", # blocked + "0", # sigignore + "0", # sigcatch + "0", # wchan + "0", # nswap + "0", # cnswap + "0", # exit_signal + "6", # processor + "0", # rt priority + "0", # policy + "7", # delayacct_blkio_ticks + ] + content = " ".join(args).encode() + with mock_open_content({f"/proc/{os.getpid()}/stat": content}): + p = psutil.Process() + assert p.name() == 'cat' + assert p.status() == psutil.STATUS_ZOMBIE + assert p.ppid() == 1 + assert p.create_time() == 6 / CLOCK_TICKS + psutil.boot_time() + cpu = p.cpu_times() + assert cpu.user == 2 / CLOCK_TICKS + assert cpu.system == 3 / CLOCK_TICKS + assert cpu.children_user == 4 / CLOCK_TICKS + assert cpu.children_system == 5 / CLOCK_TICKS + assert cpu.iowait == 7 / CLOCK_TICKS + assert p.cpu_num() == 6 + + def test_status_file_parsing(self): + content = textwrap.dedent("""\ + Uid:\t1000\t1001\t1002\t1003 + Gid:\t1004\t1005\t1006\t1007 + Threads:\t66 + Cpus_allowed:\tf + Cpus_allowed_list:\t0-7 + voluntary_ctxt_switches:\t12 + nonvoluntary_ctxt_switches:\t13""").encode() + with mock_open_content({f"/proc/{os.getpid()}/status": content}): + p = psutil.Process() + assert p.num_ctx_switches().voluntary == 12 + assert p.num_ctx_switches().involuntary == 13 + assert p.num_threads() == 66 + uids = p.uids() + assert uids.real == 1000 + assert uids.effective == 1001 + assert uids.saved == 1002 + gids = p.gids() + assert gids.real == 1004 + assert gids.effective == 1005 + assert gids.saved == 1006 + assert p._proc._get_eligible_cpus() == list(range(8)) + + def test_status_file_cpus_allowed_list(self): + content = b"Cpus_allowed_list:\t0-3,8\n" + with mock_open_content({f"/proc/{os.getpid()}/status": content}): + p = psutil.Process() + assert p._proc._get_eligible_cpus() == [0, 1, 2, 3, 8] + + def test_net_connections_enametoolong(self): + # Simulate a case where /proc/{pid}/fd/{fd} symlink points to + # a file with full path longer than PATH_MAX, see: + # https://github.com/giampaolo/psutil/issues/1940 + with mock.patch( + 'psutil._pslinux.os.readlink', + side_effect=OSError(errno.ENAMETOOLONG, ""), + ) as m: + p = psutil.Process() + with mock.patch("psutil._pslinux.debug"): + assert p.net_connections() == [] + assert m.called + + def test_create_time_monotonic(self): + p = psutil.Process() + assert p._proc.create_time() != p._proc.create_time(monotonic=True) + assert p._get_ident()[1] == p._proc.create_time(monotonic=True) + + def test_memory_extras(self): + p = psutil.Process() + with open(f"/proc/{p.pid}/status", "rb") as f: + data = f.read() + with mock.patch.object( + psutil._pslinux.Process, "_read_status_file", return_value=data + ): + mem = p.memory_extras() + vmrss = int(re.search(br"VmRSS:\s+(\d+)", data).group(1)) * 1024 + assert mem.rss_anon + mem.rss_file + mem.rss_shmem == vmrss + + def test_rlimit_infinity_normalized(self): + # Python 3.15 changed resource.prlimit() to return RLIM_INFINITY + # as the unsigned 2**64-1 instead of -1; psutil maps it back. + unsigned = 2**64 - 1 + p = psutil.Process() + with mock.patch( + "psutil._pslinux.resource.prlimit", + return_value=(unsigned, unsigned), + ) as m: + soft, hard = p.rlimit(psutil.RLIMIT_FSIZE) + assert m.called + assert soft == psutil.RLIM_INFINITY + assert hard == psutil.RLIM_INFINITY + + +class TestProcessAgainstStatus(LinuxTestCase): + """/proc/pid/stat and /proc/pid/status have many values in common. + Whenever possible, psutil uses /proc/pid/stat (it's faster). + For all those cases we check that the value found in + /proc/pid/stat (by psutil) matches the one found in + /proc/pid/status. + """ + + @classmethod + def setUpClass(cls): + cls.proc = psutil.Process() + + def read_status_file(self, linestart): + with psutil._psplatform.open_text( + f"/proc/{self.proc.pid}/status" + ) as f: + for line in f: + line = line.strip() + if line.startswith(linestart): + value = line.partition('\t')[2] + try: + return int(value) + except ValueError: + return value + raise ValueError(f"can't find {linestart!r}") + + def test_name(self): + value = self.read_status_file("Name:") + assert self.proc.name() == value + + def test_status(self): + value = self.read_status_file("State:") + value = value[value.find('(') + 1 : value.rfind(')')] + value = value.replace(' ', '-') + assert self.proc.status() == value + + def test_ppid(self): + value = self.read_status_file("PPid:") + assert self.proc.ppid() == value + + def test_num_threads(self): + value = self.read_status_file("Threads:") + assert self.proc.num_threads() == value + + def test_uids(self): + value = self.read_status_file("Uid:") + value = tuple(map(int, value.split()[1:4])) + assert self.proc.uids() == value + + def test_gids(self): + value = self.read_status_file("Gid:") + value = tuple(map(int, value.split()[1:4])) + assert self.proc.gids() == value + + @retry_on_failure + def test_num_ctx_switches(self): + value = self.read_status_file("voluntary_ctxt_switches:") + assert self.proc.num_ctx_switches().voluntary == value + value = self.read_status_file("nonvoluntary_ctxt_switches:") + assert self.proc.num_ctx_switches().involuntary == value + + def test_cpu_affinity(self): + value = self.read_status_file("Cpus_allowed_list:") + if '-' in str(value): + # The mask covers the possible CPUs, while sched_getaffinity + # only reports the online ones. + with open("/sys/devices/system/cpu/online") as f: + online = f.read().strip() + omin, omax = map(int, online.split('-')) + min_, max_ = map(int, value.split('-')) + expected = [x for x in range(min_, max_ + 1) if omin <= x <= omax] + assert self.proc.cpu_affinity() == expected + + def test_cpu_affinity_eligible_cpus(self): + value = self.read_status_file("Cpus_allowed_list:") + with mock.patch("psutil._pslinux.per_cpu_times") as m: + cpus = self.proc._proc._get_eligible_cpus() + assert cpus == _parse_cpulist(str(value)) + assert not m.called + + +# ===================================================================== +# --- test utils +# ===================================================================== + + +class TestUtils(LinuxTestCase): + def test_readlink(self): + with mock.patch("os.readlink", return_value="foo (deleted)") as m: + assert psutil._psplatform.readlink("bar") == "foo" + assert m.called diff --git a/tests/test_memleaks.py b/tests/test_memleaks.py new file mode 100755 index 0000000000..f00e413da4 --- /dev/null +++ b/tests/test_memleaks.py @@ -0,0 +1,589 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Regression test suite for detecting memory leaks in the underlying C +extension. Requires https://github.com/giampaolo/psleak. +""" + +import inspect +import os + +from psleak import LeakTest +from psleak import MemoryLeakTestCase + +import psutil +from psutil import FREEBSD +from psutil import LINUX +from psutil import MACOS +from psutil import OPENBSD +from psutil import POSIX +from psutil import SUNOS +from psutil import WINDOWS +from psutil import _psutil + +from . import HAS_CPU_FREQ +from . import HAS_HEAP_INFO +from . import HAS_NET_IO_COUNTERS +from . import HAS_PROC_CPU_AFFINITY +from . import HAS_PROC_CPU_NUM +from . import HAS_PROC_ENVIRON +from . import HAS_PROC_IO_COUNTERS +from . import HAS_PROC_IONICE +from . import HAS_PROC_MEMORY_EXTRAS +from . import HAS_PROC_MEMORY_FOOTPRINT +from . import HAS_PROC_MEMORY_MAPS +from . import HAS_PROC_RLIMIT +from . import HAS_SENSORS_BATTERY +from . import HAS_SENSORS_FANS +from . import HAS_SENSORS_TEMPERATURES +from . import PYTEST_PARALLEL +from . import create_sockets +from . import get_testfn +from . import process_namespace +from . import pytest +from . import skip_on_access_denied +from . import skipif +from . import spawn_subproc +from . import system_namespace +from . import terminate + +thisproc = psutil.Process() + + +MemoryLeakTestCase.retries = 30 # minimize false positives + +# Be quiet when running under xdist. +MemoryLeakTestCase.verbosity = 0 if PYTEST_PARALLEL else 1 + +TIMES = MemoryLeakTestCase.times +FEW_TIMES = int(TIMES / 10) + + +# =================================================================== +# Process class +# =================================================================== + + +class TestProcess(MemoryLeakTestCase): + """Test leaks of Process class methods.""" + + proc = thisproc + + def test_coverage(self): + ns = process_namespace(None) + ns.test_class_coverage(self, ns.getters + ns.setters) + + def test_name(self): + self.execute(self.proc.name) + + def test_cmdline(self): + if WINDOWS and self.proc.is_running(): + self.proc.cmdline() + self.execute(self.proc.cmdline) + + def test_exe(self): + self.execute(self.proc.exe) + + def test_ppid(self): + self.execute(self.proc.ppid) + + @skipif(not POSIX, reason="POSIX only") + def test_uids(self): + self.execute(self.proc.uids) + + @skipif(not POSIX, reason="POSIX only") + def test_gids(self): + self.execute(self.proc.gids) + + def test_status(self): + self.execute(self.proc.status) + + def test_nice(self): + self.execute(self.proc.nice) + + def test_nice_set(self): + niceness = thisproc.nice() + self.execute(lambda: self.proc.nice(niceness)) + + @skipif(not HAS_PROC_IONICE, reason="not supported") + def test_ionice(self): + self.execute(self.proc.ionice) + + @skipif(not HAS_PROC_IONICE, reason="not supported") + def test_ionice_set(self): + if WINDOWS: + value = thisproc.ionice() + self.execute(lambda: self.proc.ionice(value)) + else: + self.execute(lambda: self.proc.ionice(psutil.IOPRIO_CLASS_NONE)) + + @skipif(not HAS_PROC_IO_COUNTERS, reason="not supported") + def test_io_counters(self): + self.execute(self.proc.io_counters) + + @skipif(POSIX, reason="worthless on POSIX") + def test_username(self): + # always open 1 handle on Windows (only once) + psutil.Process().username() + self.execute(self.proc.username) + + def test_create_time(self): + self.execute(self.proc.create_time) + + @skip_on_access_denied(only_if=OPENBSD) + def test_num_threads(self): + self.execute(self.proc.num_threads) + + @skipif(not WINDOWS, reason="WINDOWS only") + def test_num_handles(self): + self.execute(self.proc.num_handles) + + @skipif(not POSIX, reason="POSIX only") + def test_num_fds(self): + self.execute(self.proc.num_fds) + + def test_num_ctx_switches(self): + self.execute(self.proc.num_ctx_switches) + + @skip_on_access_denied(only_if=OPENBSD) + def test_threads(self): + self.execute(self.proc.threads, times=50 if WINDOWS else TIMES) + + def test_cpu_times(self): + self.execute(self.proc.cpu_times) + + @skipif(not HAS_PROC_CPU_NUM, reason="not supported") + def test_cpu_num(self): + self.execute(self.proc.cpu_num) + + def test_memory_info(self): + self.execute(self.proc.memory_info) + + @skipif(not HAS_PROC_MEMORY_EXTRAS, reason="not supported") + def test_memory_extras(self): + self.execute(self.proc.memory_extras) + + @skipif(not HAS_PROC_MEMORY_FOOTPRINT, reason="not supported") + def test_memory_footprint(self): + self.execute(self.proc.memory_footprint, times=50) # slow + + @skipif(not POSIX, reason="POSIX only") + def test_terminal(self): + self.execute(self.proc.terminal) + + def test_resume(self): + times = FEW_TIMES if POSIX else self.times + self.execute(self.proc.resume, times=times) + + def test_cwd(self): + self.execute(self.proc.cwd) + + @skipif(not HAS_PROC_CPU_AFFINITY, reason="not supported") + def test_cpu_affinity(self): + self.execute(self.proc.cpu_affinity) + + @skipif(not HAS_PROC_CPU_AFFINITY, reason="not supported") + def test_cpu_affinity_set(self): + affinity = thisproc.cpu_affinity() + self.execute(lambda: self.proc.cpu_affinity(affinity)) + + def test_open_files(self): + with open(get_testfn(), 'w'): + self.execute(self.proc.open_files) + + @skipif(not HAS_PROC_MEMORY_MAPS, reason="not supported") + @skipif(LINUX, reason="too slow on LINUX") + def test_memory_maps(self): + self.execute(self.proc.memory_maps, times=60, retries=10) + + def test_page_faults(self): + self.execute(self.proc.page_faults) + + @skipif(not LINUX, reason="LINUX only") + @skipif(not HAS_PROC_RLIMIT, reason="not supported") + def test_rlimit(self): + self.execute(lambda: self.proc.rlimit(psutil.RLIMIT_NOFILE)) + + @skipif(not LINUX, reason="LINUX only") + @skipif(not HAS_PROC_RLIMIT, reason="not supported") + def test_rlimit_set(self): + limit = thisproc.rlimit(psutil.RLIMIT_NOFILE) + self.execute(lambda: self.proc.rlimit(psutil.RLIMIT_NOFILE, limit)) + + # Windows implementation is based on a single system-wide + # function (tested later). + @skipif(WINDOWS, reason="worthless on WINDOWS") + @skipif(LINUX, reason="pure python, too slow") + @skipif(SUNOS, reason="parses pfiles CLI") + def test_net_connections(self): + with create_sockets(): + kind = 'inet' if SUNOS else 'all' + self.execute(lambda: self.proc.net_connections(kind)) + + @skipif(not HAS_PROC_ENVIRON, reason="not supported") + def test_environ(self): + self.execute(self.proc.environ) + + @skipif(not WINDOWS, reason="WINDOWS only") + def test_proc_oneshot(self): + self.execute(lambda: _psutil.proc_oneshot(os.getpid())) + + +class TestTerminatedProcess(TestProcess): + """Repeat the tests above looking for leaks occurring when dealing + with terminated processes raising NoSuchProcess exception. + The C functions are still invoked but will follow different code + paths. We'll check those code paths. + """ + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.subp = spawn_subproc() + cls.proc = psutil.Process(cls.subp.pid) + cls.proc.kill() + cls.proc.wait() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + terminate(cls.subp) + + def call(self, fun): + try: + fun() + except psutil.NoSuchProcess: + pass + + if WINDOWS: + + def test_kill(self): + self.execute(self.proc.kill) + + def test_terminate(self): + self.execute(self.proc.terminate) + + def test_suspend(self): + self.execute(self.proc.suspend) + + def test_resume(self): + self.execute(self.proc.resume) + + def test_wait(self): + self.execute(self.proc.wait) + + def test_proc_oneshot(self): + # test dual implementation + def call(): + try: + return _psutil.proc_oneshot(self.proc.pid) + except ProcessLookupError: + pass + + self.execute(call) + + +@skipif(not WINDOWS, reason="WINDOWS only") +class TestProcessDualImplementation(MemoryLeakTestCase): + def test_cmdline_peb_true(self): + # The first CommandLineToArgvW() call loads shell32, leaving + # persistent handles. + _psutil.proc_cmdline(os.getpid(), use_peb=True) + self.execute(lambda: _psutil.proc_cmdline(os.getpid(), use_peb=True)) + + def test_cmdline_peb_false(self): + _psutil.proc_cmdline(os.getpid(), use_peb=False) # prime (see above) + self.execute(lambda: _psutil.proc_cmdline(os.getpid(), use_peb=False)) + + +# =================================================================== +# system APIs +# =================================================================== + + +class TestModuleFunctions(MemoryLeakTestCase): + """Test leaks of psutil module functions.""" + + def test_coverage(self): + ns = system_namespace() + ns.test_class_coverage(self, ns.all) + + # --- cpu + + def test_cpu_count(self): # logical + self.execute(lambda: psutil.cpu_count(logical=True)) + + def test_cpu_count_cores(self): + self.execute(lambda: psutil.cpu_count(logical=False)) + + def test_cpu_times(self): + self.execute(psutil.cpu_times) + + def test_per_cpu_times(self): + self.execute(lambda: psutil.cpu_times(percpu=True)) + + def test_cpu_stats(self): + self.execute(psutil.cpu_stats) + + @skipif(not HAS_CPU_FREQ, reason="not supported") + def test_cpu_freq(self): + times = FEW_TIMES if LINUX else self.times + self.execute(psutil.cpu_freq, times=times) + + @skipif(not WINDOWS, reason="WINDOWS only") + def test_getloadavg(self): + psutil.getloadavg() + self.execute(psutil.getloadavg) + + # --- mem + + def test_virtual_memory(self): + self.execute(psutil.virtual_memory) + + # TODO: remove this skip when this gets fixed + @skipif(SUNOS, reason="worthless on SUNOS (uses a subprocess)") + def test_swap_memory(self): + self.execute(psutil.swap_memory) + + def test_pid_exists(self): + times = FEW_TIMES if POSIX else self.times + self.execute(lambda: psutil.pid_exists(os.getpid()), times=times) + + # --- disk + + def test_disk_usage(self): + times = FEW_TIMES if POSIX else self.times + self.execute(lambda: psutil.disk_usage('.'), times=times) + + def test_disk_partitions(self): + self.execute(psutil.disk_partitions) + + @skipif( + LINUX and not os.path.exists('/proc/diskstats'), + reason="/proc/diskstats not available on this Linux version", + ) + def test_disk_io_counters(self): + self.execute(lambda: psutil.disk_io_counters(nowrap=False)) + + # --- proc + + def test_pids(self): + self.execute(psutil.pids) + + # --- net + + @skipif(not HAS_NET_IO_COUNTERS, reason="not supported") + def test_net_io_counters(self): + self.execute(lambda: psutil.net_io_counters(nowrap=False)) + + @skipif(MACOS and os.getuid() != 0, reason="need root access") + @skipif(LINUX, reason="pure python, too slow") + def test_net_connections(self): + # slow + with create_sockets(): + psutil.net_connections(kind='all') + self.execute( + lambda: psutil.net_connections(kind='all'), times=TIMES / 2 + ) + + def test_net_if_addrs(self): + psutil.net_if_addrs() # XXX prime + # Note: verified that on Windows this was a false positive. + tolerance = 80 * 1024 if WINDOWS else self.tolerance + self.execute(psutil.net_if_addrs, tolerance=tolerance) + + def test_net_if_stats(self): + self.execute(psutil.net_if_stats) + + # --- sensors + + @skipif(not HAS_SENSORS_BATTERY, reason="not supported") + def test_sensors_battery(self): + self.execute(psutil.sensors_battery) + + @skipif(not HAS_SENSORS_TEMPERATURES, reason="not supported") + @skipif(LINUX, reason="too slow on LINUX") + def test_sensors_temperatures(self): + times = FEW_TIMES if LINUX else self.times + self.execute(psutil.sensors_temperatures, times=times) + + @skipif(not HAS_SENSORS_FANS, reason="not supported") + def test_sensors_fans(self): + times = FEW_TIMES if LINUX else self.times + self.execute(psutil.sensors_fans, times=times) + + # --- others + + def test_boot_time(self): + self.execute(psutil.boot_time) + + def test_users(self): + self.execute(psutil.users) + + def test_set_debug(self): + self.execute(lambda: psutil._set_debug(False)) + + @skipif(not HAS_HEAP_INFO, reason="not supported") + def test_heap_info(self): + self.execute(psutil.heap_info, times=25 if WINDOWS else TIMES) # slow + + @skipif(not HAS_HEAP_INFO, reason="not supported") + def test_heap_trim(self): + self.execute(psutil.heap_trim) + + if WINDOWS: + + # --- win services + + def test_win_service_iter(self): + self.execute(_psutil.winservice_enumerate) + + def test_win_service_get(self): + pass + + def test_win_service_get_config(self): + name = next(psutil.win_service_iter()).name() + self.execute(lambda: _psutil.winservice_query_config(name)) + + def test_win_service_get_status(self): + name = next(psutil.win_service_iter()).name() + self.execute(lambda: _psutil.winservice_query_status(name)) + + def test_win_service_get_description(self): + name = next(psutil.win_service_iter()).name() + self.execute(lambda: _psutil.winservice_query_descr(name)) + + +# =================================================================== +# bad arguments +# =================================================================== + + +class TestBadargs(MemoryLeakTestCase): + """Pass a bad argument to each C function that accepts one, and make + sure the resulting `PyArg_ParseTuple` failure path doesn't leak + memory: a C function may allocate its return object before parsing + its arguments, see: https://github.com/giampaolo/psutil/pull/2857/. + """ + + retries = 20 + badargs = (object(),) * 20 + + def execute(self, fun, *args, **kwargs): + # Every function here is called with a bad arg and must raise + # TypeError (the parse-failure path we check for leaks). + def call(): + try: + fun(*args) + except TypeError: + pass + else: + pytest.fail(f"{fun} did not raise TypeError") + + super().execute(call, **kwargs) + + @classmethod + def cext_arg_funcs(cls): + # Names of the C functions that actually parse at least one + # argument. + names = [] + for name in dir(_psutil): + if name.startswith("_"): + continue + func = getattr(_psutil, name) + if not inspect.isbuiltin(func): # skip exception classes + continue + try: + func(*cls.badargs) + except NotImplementedError: + pass + except TypeError as err: + # "takes no arguments" => METH_NOARGS (no arg to test). + if "takes no arguments" not in str(err): + names.append(name) + return names + + @classmethod + def auto_generate(cls): + return { + name: LeakTest(getattr(_psutil, name), *cls.badargs) + for name in cls.cext_arg_funcs() + } + + +def cext_has(name): + return skipif(not hasattr(_psutil, name), reason=f"no _psutil.{name}()") + + +class TestBadargs2(MemoryLeakTestCase): + """Like TestBadargsMemleaks, but for the error paths reached + *after* `PyArg_ParseTuple` succeeds, by passing an invalid arg like + a negative PID, a bogus NIC name etc., making the C function fail + deeper. + """ + + retries = 20 + pid = os.getpid() + + # --- portable: same signature and error everywhere they exist + + @cext_has("proc_priority_get") + def test_proc_priority_get(self): + self.execute_w_exc(OSError, _psutil.proc_priority_get, -1) + + @cext_has("proc_priority_set") + def test_proc_priority_set(self): + self.execute_w_exc(OSError, _psutil.proc_priority_set, -1, 0) + + @cext_has("proc_cpu_affinity_get") + def test_proc_cpu_affinity_get(self): + # FreeBSD's cpuset_getaffinity() reads -1 as "the current + # process", so it succeeds. Use another negative PID. + pid = -2 if FREEBSD else -1 + self.execute_w_exc(OSError, _psutil.proc_cpu_affinity_get, pid) + + @cext_has("net_if_flags") + def test_net_if_flags(self): + self.execute_w_exc(OSError, _psutil.net_if_flags, "nonexistent0") + + @cext_has("net_if_mtu") + def test_net_if_mtu(self): + self.execute_w_exc(OSError, _psutil.net_if_mtu, "nonexistent0") + + @cext_has("net_if_is_running") + def test_net_if_is_running(self): + self.execute_w_exc(OSError, _psutil.net_if_is_running, "nonexistent0") + + # --- Linux only + + @cext_has("proc_ioprio_set") + def test_proc_ioprio_set(self): + self.execute_w_exc(OSError, _psutil.proc_ioprio_set, self.pid, -1, 0) + + @cext_has("proc_ioprio_get") + def test_proc_ioprio_get(self): + self.execute_w_exc(OSError, _psutil.proc_ioprio_get, -1) + + @skipif(not LINUX, reason="LINUX only") + def test_disk_partitions(self): + self.execute_w_exc(OSError, _psutil.disk_partitions, "/does/not/exist") + + @skipif(not LINUX, reason="LINUX only") + def test_net_if_duplex_speed(self): + self.execute_w_exc( + OSError, _psutil.net_if_duplex_speed, "nonexistent0" + ) + + # --- other platform-specific behavior + + @skipif(not LINUX, reason="LINUX only") + def test_proc_cpu_affinity_set(self): + self.execute_w_exc( + ValueError, _psutil.proc_cpu_affinity_set, self.pid, [-1] + ) + + def test_check_pid_range(self): + self.execute_w_exc(ValueError, _psutil.check_pid_range, -1) diff --git a/tests/test_misc.py b/tests/test_misc.py new file mode 100755 index 0000000000..2b0386642f --- /dev/null +++ b/tests/test_misc.py @@ -0,0 +1,883 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Miscellaneous tests.""" + +import collections +import contextlib +import io +import json +import os +import pickle +import socket +import subprocess +import sys +import textwrap +import warnings +from unittest import mock + +import psutil +from psutil import POSIX +from psutil import WINDOWS +from psutil import _psutil +from psutil._common import bcat +from psutil._common import broadcast_addr +from psutil._common import cat +from psutil._common import debug +from psutil._common import isfile_strict +from psutil._common import memoize_when_activated +from psutil._common import parse_environ_block +from psutil._common import supports_ipv6 +from psutil._common import warn +from psutil._common import wrap_numbers +from psutil._ntuples import snicaddr + +from . import HAS_NET_IO_COUNTERS +from . import ROOT_DIR +from . import PsutilTestCase +from . import import_module_by_path +from . import process_namespace +from . import pytest +from . import reload_module +from . import skipif +from . import system_namespace + +# =================================================================== +# --- Test classes' repr(), str(), ... +# =================================================================== + + +class TestSpecialMethods(PsutilTestCase): + def test_check_pid_range(self): + with pytest.raises(OverflowError): + _psutil.check_pid_range(2**128) + with pytest.raises(psutil.NoSuchProcess): + psutil.Process(2**128) + + def test_process__repr__(self, func=repr): + p = psutil.Process(self.spawn_subproc().pid) + r = func(p) + assert "psutil.Process" in r + assert f"pid={p.pid}" in r + assert f"name='{p.name()}'" in r.replace("name=u'", "name='") + assert "status=" in r + assert "exitcode=" not in r + p.terminate() + p.wait() + r = func(p) + assert "status='terminated'" in r + assert "exitcode=" in r + + with mock.patch.object( + psutil.Process, + "name", + side_effect=psutil.ZombieProcess(os.getpid()), + ): + p = psutil.Process() + r = func(p) + assert f"pid={p.pid}" in r + assert "status='zombie'" in r + assert "name=" not in r + with mock.patch.object( + psutil.Process, + "name", + side_effect=psutil.NoSuchProcess(os.getpid()), + ): + p = psutil.Process() + r = func(p) + assert f"pid={p.pid}" in r + assert "terminated" in r + assert "name=" not in r + with mock.patch.object( + psutil.Process, + "name", + side_effect=psutil.AccessDenied(os.getpid()), + ): + p = psutil.Process() + r = func(p) + assert f"pid={p.pid}" in r + assert "name=" not in r + + def test_process__str__(self): + self.test_process__repr__(func=str) + + def test_error__repr__(self): + assert repr(psutil.Error()) == "psutil.Error()" + + def test_error__str__(self): + assert str(psutil.Error()) == "" + + def test_no_such_process__repr__(self): + assert ( + repr(psutil.NoSuchProcess(321)) + == "psutil.NoSuchProcess(pid=321, msg='process no longer exists')" + ) + assert ( + repr(psutil.NoSuchProcess(321, name="name", msg="msg")) + == "psutil.NoSuchProcess(pid=321, name='name', msg='msg')" + ) + + def test_no_such_process__str__(self): + assert ( + str(psutil.NoSuchProcess(321)) + == "process no longer exists (pid=321)" + ) + assert ( + str(psutil.NoSuchProcess(321, name="name", msg="msg")) + == "msg (pid=321, name='name')" + ) + + def test_zombie_process__repr__(self): + assert ( + repr(psutil.ZombieProcess(321)) + == 'psutil.ZombieProcess(pid=321, msg="PID still ' + 'exists but it\'s a zombie")' + ) + assert ( + repr(psutil.ZombieProcess(321, name="name", ppid=320, msg="foo")) + == "psutil.ZombieProcess(pid=321, ppid=320, name='name'," + " msg='foo')" + ) + + def test_zombie_process__str__(self): + assert ( + str(psutil.ZombieProcess(321)) + == "PID still exists but it's a zombie (pid=321)" + ) + assert ( + str(psutil.ZombieProcess(321, name="name", ppid=320, msg="foo")) + == "foo (pid=321, ppid=320, name='name')" + ) + + def test_access_denied__repr__(self): + assert repr(psutil.AccessDenied(321)) == "psutil.AccessDenied(pid=321)" + assert ( + repr(psutil.AccessDenied(321, name="name", msg="msg")) + == "psutil.AccessDenied(pid=321, name='name', msg='msg')" + ) + + def test_access_denied__str__(self): + assert str(psutil.AccessDenied(321)) == "(pid=321)" + assert ( + str(psutil.AccessDenied(321, name="name", msg="msg")) + == "msg (pid=321, name='name')" + ) + + def test_timeout_expired__repr__(self): + assert ( + repr(psutil.TimeoutExpired(5)) + == "psutil.TimeoutExpired(seconds=5, msg='timeout after 5" + " seconds')" + ) + assert ( + repr(psutil.TimeoutExpired(5, pid=321, name="name")) + == "psutil.TimeoutExpired(pid=321, name='name', seconds=5, " + "msg='timeout after 5 seconds')" + ) + + def test_timeout_expired__str__(self): + assert str(psutil.TimeoutExpired(5)) == "timeout after 5 seconds" + assert ( + str(psutil.TimeoutExpired(5, pid=321, name="name")) + == "timeout after 5 seconds (pid=321, name='name')" + ) + + +# =================================================================== +# --- Misc, generic, corner cases +# =================================================================== + + +class TestMisc(PsutilTestCase): + def test__all__(self): + dir_psutil = dir(psutil) + # assert there's no duplicates + assert len(dir_psutil) == len(set(dir_psutil)) + for name in dir_psutil: + if name in { + 'debug', + 'warn', + 'tests', + 'test', + 'PermissionError', + 'ProcessLookupError', + }: + continue + if not name.startswith('_'): + try: + __import__(name) + except ImportError: + if name not in psutil.__all__: + fun = getattr(psutil, name) + if fun is None: + continue + if ( + fun.__doc__ is not None + and 'deprecated' not in fun.__doc__.lower() + ): + return pytest.fail( + f"{name!r} not in psutil.__all__" + ) + + # Import 'star' will break if __all__ is inconsistent, see: + # https://github.com/giampaolo/psutil/issues/656 + # Can't do `from psutil import *` as it won't work + # so we simply iterate over __all__. + for name in psutil.__all__: + assert name in dir_psutil + + def test_version(self): + assert ( + '.'.join([str(x) for x in psutil.version_info]) + == psutil.__version__ + ) + + def test_process_as_dict_no_new_names(self): + # See https://github.com/giampaolo/psutil/issues/813 + p = psutil.Process() + p.foo = '1' + assert 'foo' not in p.as_dict() + + def test_serialization(self): + def check(ret): + json.loads(json.dumps(ret)) + + a = pickle.dumps(ret) + b = pickle.loads(a) + assert ret == b + + # --- process APIs + + proc = psutil.Process() + check(psutil.Process().as_dict()) + + ns = process_namespace(proc) + for fun, name in ns.iter(ns.getters, clear_cache=True): + with self.subTest(proc=str(proc), name=name): + try: + ret = fun() + except psutil.Error: + pass + else: + check(ret) + + # --- system APIs + + ns = system_namespace() + for fun, name in ns.iter(ns.getters): + if name in {"win_service_iter", "win_service_get"}: + continue + with self.subTest(name=name): + try: + ret = fun() + except psutil.AccessDenied: + pass + else: + check(ret) + + # --- exception classes + + b = pickle.loads( + pickle.dumps( + psutil.NoSuchProcess(pid=4567, name='name', msg='msg') + ) + ) + assert isinstance(b, psutil.NoSuchProcess) + assert b.pid == 4567 + assert b.name == 'name' + assert b.msg == 'msg' + + b = pickle.loads( + pickle.dumps( + psutil.ZombieProcess(pid=4567, name='name', ppid=42, msg='msg') + ) + ) + assert isinstance(b, psutil.ZombieProcess) + assert b.pid == 4567 + assert b.ppid == 42 + assert b.name == 'name' + assert b.msg == 'msg' + + b = pickle.loads( + pickle.dumps(psutil.AccessDenied(pid=123, name='name', msg='msg')) + ) + assert isinstance(b, psutil.AccessDenied) + assert b.pid == 123 + assert b.name == 'name' + assert b.msg == 'msg' + + b = pickle.loads( + pickle.dumps( + psutil.TimeoutExpired(seconds=33, pid=4567, name='name') + ) + ) + assert isinstance(b, psutil.TimeoutExpired) + assert b.seconds == 33 + assert b.pid == 4567 + assert b.name == 'name' + + def test_sanity_version_check(self): + # see: https://github.com/giampaolo/psutil/issues/564 + with mock.patch.object(_psutil, "version", return_value="0.0.0"): + with pytest.raises(ImportError) as cm: + reload_module(psutil) + assert "version conflict" in str(cm.value).lower() + + def test_reload_keeps_all(self): + # A reload reuses the module dict, so the enum constants are + # already there and used to not make it back into __all__. + before = sorted(psutil.__all__) + reload_module(psutil) + assert sorted(psutil.__all__) == before + + +# =================================================================== +# --- C extension +# =================================================================== + + +class TestCExtension(PsutilTestCase): + + def test_exceptions_survive_reimport(self): + # PEP 489 multi-phase init re-runs the C exec slot on re-import; + # the C exceptions are cached in process-global vars so their + # identity survives. Run in a subprocess: re-importing the + # module (del from sys.modules + import) mutates global state and + # would leak into other tests. + attrs = ( + ["TimeoutExpired", "TimeoutAbandoned"] + if WINDOWS + else ["ZombieProcessError"] + ) + code = textwrap.dedent(f""" + import importlib + import sys + + from psutil import _psutil + + attrs = {attrs!r} + before = {{a: getattr(_psutil, a) for a in attrs}} + del sys.modules[_psutil.__name__] + new = importlib.import_module(_psutil.__name__) + for a in attrs: + assert getattr(new, a) is before[a], a + """) + subprocess.check_output( + [sys.executable, "-c", code], stderr=subprocess.STDOUT + ) + + +# =================================================================== +# --- psutil/_common.py utils +# =================================================================== + + +class TestCommonModule(PsutilTestCase): + def test_memoize_when_activated(self): + class Foo: + @memoize_when_activated + def foo(self): + calls.append(None) + + f = Foo() + calls = [] + f.foo() + f.foo() + assert len(calls) == 2 + + # activate + calls = [] + f.foo.cache_activate(f) + f.foo() + f.foo() + assert len(calls) == 1 + + # deactivate + calls = [] + f.foo.cache_deactivate(f) + f.foo() + f.foo() + assert len(calls) == 2 + + def test_parse_environ_block(self): + def k(s): + return s.upper() if WINDOWS else s + + assert parse_environ_block("a=1\0") == {k("a"): "1"} + assert parse_environ_block("a=1\0b=2\0\0") == { + k("a"): "1", + k("b"): "2", + } + assert parse_environ_block("a=1\0b=\0\0") == {k("a"): "1", k("b"): ""} + # ignore everything after \0\0 + assert parse_environ_block("a=1\0b=2\0\0c=3\0") == { + k("a"): "1", + k("b"): "2", + } + # ignore everything that is not an assignment + assert parse_environ_block("xxx\0a=1\0") == {k("a"): "1"} + assert parse_environ_block("a=1\0=b=2\0") == {k("a"): "1"} + # do not fail if the block is incomplete + assert parse_environ_block("a=1\0b=2") == {k("a"): "1"} + + def test_supports_ipv6(self): + if supports_ipv6(): + with mock.patch('psutil._common.socket') as s: + s.has_ipv6 = False + assert not supports_ipv6() + + with mock.patch( + 'psutil._common.socket.socket', side_effect=OSError + ) as s: + assert not supports_ipv6() + assert s.called + + with mock.patch( + 'psutil._common.socket.socket', side_effect=socket.gaierror + ) as s: + assert not supports_ipv6() + assert s.called + + with mock.patch( + 'psutil._common.socket.socket.bind', + side_effect=socket.gaierror, + ) as s: + assert not supports_ipv6() + assert s.called + else: + with pytest.raises(OSError): + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + try: + sock.bind(("::1", 0)) + finally: + sock.close() + + def test_isfile_strict(self): + this_file = os.path.abspath(__file__) + assert isfile_strict(this_file) + assert not isfile_strict(os.path.dirname(this_file)) + with mock.patch('psutil._common.os.stat', side_effect=PermissionError): + with pytest.raises(OSError): + isfile_strict(this_file) + with mock.patch( + 'psutil._common.os.stat', side_effect=FileNotFoundError + ): + assert not isfile_strict(this_file) + with mock.patch('psutil._common.stat.S_ISREG', return_value=False): + assert not isfile_strict(this_file) + + def test_debug(self): + with mock.patch.object(psutil._common, "PSUTIL_DEBUG", True): + with contextlib.redirect_stderr(io.StringIO()) as f: + debug("hello") + sys.stderr.flush() + msg = f.getvalue() + assert msg.startswith("psutil-debug"), msg + assert "hello" in msg + assert __file__.replace('.pyc', '.py') in msg + + # supposed to use repr(exc) + with mock.patch.object(psutil._common, "PSUTIL_DEBUG", True): + with contextlib.redirect_stderr(io.StringIO()) as f: + debug(ValueError("this is an error")) + msg = f.getvalue() + assert "ignoring ValueError" in msg + assert "'this is an error'" in msg + + # supposed to use str(exc), because of extra info about file name + with mock.patch.object(psutil._common, "PSUTIL_DEBUG", True): + with contextlib.redirect_stderr(io.StringIO()) as f: + exc = OSError(2, "no such file") + exc.filename = "/foo" + debug(exc) + msg = f.getvalue() + assert "no such file" in msg + assert "/foo" in msg + + def test_warn(self): + with mock.patch.object(psutil._common, "PSUTIL_TESTING", True): + with pytest.raises(RuntimeError, match="CRITICAL: hello"): + warn("hello") + + with mock.patch.object(psutil._common, "PSUTIL_TESTING", False): + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") + warn("hello") + assert len(ws) == 1 + assert ws[0].category is RuntimeWarning + assert "hello" in str(ws[0].message) + assert __file__.replace('.pyc', '.py') in str(ws[0].message) + + def test_cat_bcat(self): + testfn = self.get_testfn() + with open(testfn, "w") as f: + f.write("foo") + assert cat(testfn) == "foo" + assert bcat(testfn) == b"foo" + with pytest.raises(FileNotFoundError): + cat(testfn + '-invalid') + with pytest.raises(FileNotFoundError): + bcat(testfn + '-invalid') + assert cat(testfn + '-invalid', fallback="bar") == "bar" + assert bcat(testfn + '-invalid', fallback="bar") == "bar" + + def test_broadcast_addr(self): + def addr(address, netmask): + return snicaddr(socket.AF_INET, address, netmask, None, None) + + assert ( + broadcast_addr(addr("10.1.1.86", "255.255.255.0")) == "10.1.1.255" + ) + assert ( + broadcast_addr(addr("172.20.10.7", "255.255.255.240")) + == "172.20.10.15" + ) + + def test_broadcast_addr_single_host(self): + # A /32 is a single-host network, it has no broadcast address. + nt = snicaddr( + socket.AF_INET, "89.234.156.160", "255.255.255.255", None, None + ) + assert broadcast_addr(nt) is None + + +class TestBytes2Human(PsutilTestCase): + + def test_basic(self): + assert psutil.bytes2human(0) == "0.0B" + assert psutil.bytes2human(1000) == "1000.0B" + assert psutil.bytes2human(10000) == "9.8K" + assert psutil.bytes2human(100001221) == "95.4M" + assert psutil.bytes2human(1099511627776) == "1.0T" + + +# =================================================================== +# --- Tests for wrap_numbers() function. +# =================================================================== + + +nt = collections.namedtuple('foo', 'a b c') + + +class TestWrapNumbers(PsutilTestCase): + def setUp(self): + wrap_numbers.cache_clear() + + tearDown = setUp + + def test_first_call(self): + input = {'disk1': nt(5, 5, 5)} + assert wrap_numbers(input, 'disk_io') == input + + def test_input_hasnt_changed(self): + input = {'disk1': nt(5, 5, 5)} + assert wrap_numbers(input, 'disk_io') == input + assert wrap_numbers(input, 'disk_io') == input + + def test_increase_but_no_wrap(self): + input = {'disk1': nt(5, 5, 5)} + assert wrap_numbers(input, 'disk_io') == input + input = {'disk1': nt(10, 15, 20)} + assert wrap_numbers(input, 'disk_io') == input + input = {'disk1': nt(20, 25, 30)} + assert wrap_numbers(input, 'disk_io') == input + input = {'disk1': nt(20, 25, 30)} + assert wrap_numbers(input, 'disk_io') == input + + def test_wrap(self): + # let's say 100 is the threshold + input = {'disk1': nt(100, 100, 100)} + assert wrap_numbers(input, 'disk_io') == input + # first wrap restarts from 10 + input = {'disk1': nt(100, 100, 10)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(100, 100, 110)} + # then it remains the same + input = {'disk1': nt(100, 100, 10)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(100, 100, 110)} + # then it goes up + input = {'disk1': nt(100, 100, 90)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(100, 100, 190)} + # then it wraps again + input = {'disk1': nt(100, 100, 20)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(100, 100, 210)} + # and remains the same + input = {'disk1': nt(100, 100, 20)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(100, 100, 210)} + # now wrap another num + input = {'disk1': nt(50, 100, 20)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(150, 100, 210)} + # and again + input = {'disk1': nt(40, 100, 20)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(190, 100, 210)} + # keep it the same + input = {'disk1': nt(40, 100, 20)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(190, 100, 210)} + + def test_changing_keys(self): + # Emulate a case where the second call to disk_io() + # (or whatever) provides a new disk, then the new disk + # disappears on the third call. + input = {'disk1': nt(5, 5, 5)} + assert wrap_numbers(input, 'disk_io') == input + input = {'disk1': nt(5, 5, 5), 'disk2': nt(7, 7, 7)} + assert wrap_numbers(input, 'disk_io') == input + input = {'disk1': nt(8, 8, 8)} + assert wrap_numbers(input, 'disk_io') == input + + def test_changing_keys_w_wrap(self): + input = {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 100)} + assert wrap_numbers(input, 'disk_io') == input + # disk 2 wraps + input = {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 10)} + assert wrap_numbers(input, 'disk_io') == { + 'disk1': nt(50, 50, 50), + 'disk2': nt(100, 100, 110), + } + # disk 2 disappears + input = {'disk1': nt(50, 50, 50)} + assert wrap_numbers(input, 'disk_io') == input + + # then it appears again; the old wrap is supposed to be + # gone. + input = {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 100)} + assert wrap_numbers(input, 'disk_io') == input + # remains the same + input = {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 100)} + assert wrap_numbers(input, 'disk_io') == input + # and then wraps again + input = {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 10)} + assert wrap_numbers(input, 'disk_io') == { + 'disk1': nt(50, 50, 50), + 'disk2': nt(100, 100, 110), + } + + def test_real_data(self): + d = { + 'nvme0n1': (300, 508, 640, 1571, 5970, 1987, 2049, 451751, 47048), + 'nvme0n1p1': (1171, 2, 5600256, 1024, 516, 0, 0, 0, 8), + 'nvme0n1p2': (54, 54, 2396160, 5165056, 4, 24, 30, 1207, 28), + 'nvme0n1p3': (2389, 4539, 5154, 150, 4828, 1844, 2019, 398, 348), + } + assert wrap_numbers(d, 'disk_io') == d + assert wrap_numbers(d, 'disk_io') == d + # decrease this ↓ + d = { + 'nvme0n1': (100, 508, 640, 1571, 5970, 1987, 2049, 451751, 47048), + 'nvme0n1p1': (1171, 2, 5600256, 1024, 516, 0, 0, 0, 8), + 'nvme0n1p2': (54, 54, 2396160, 5165056, 4, 24, 30, 1207, 28), + 'nvme0n1p3': (2389, 4539, 5154, 150, 4828, 1844, 2019, 398, 348), + } + out = wrap_numbers(d, 'disk_io') + assert out['nvme0n1'][0] == 400 + + # --- cache tests + + def test_cache_first_call(self): + input = {'disk1': nt(5, 5, 5)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + assert cache[1] == {'disk_io': {}} + assert cache[2] == {'disk_io': {}} + + def test_cache_call_twice(self): + input = {'disk1': nt(5, 5, 5)} + wrap_numbers(input, 'disk_io') + input = {'disk1': nt(10, 10, 10)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + assert cache[1] == { + 'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 0} + } + assert cache[2] == {'disk_io': {}} + + def test_cache_wrap(self): + # let's say 100 is the threshold + input = {'disk1': nt(100, 100, 100)} + wrap_numbers(input, 'disk_io') + + # first wrap restarts from 10 + input = {'disk1': nt(100, 100, 10)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + assert cache[1] == { + 'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 100} + } + assert cache[2] == {'disk_io': {'disk1': {('disk1', 2)}}} + + def check_cache_info(): + cache = wrap_numbers.cache_info() + assert cache[1] == { + 'disk_io': { + ('disk1', 0): 0, + ('disk1', 1): 0, + ('disk1', 2): 100, + } + } + assert cache[2] == {'disk_io': {'disk1': {('disk1', 2)}}} + + # then it remains the same + input = {'disk1': nt(100, 100, 10)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + check_cache_info() + + # then it goes up + input = {'disk1': nt(100, 100, 90)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + check_cache_info() + + # then it wraps again + input = {'disk1': nt(100, 100, 20)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + assert cache[1] == { + 'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 190} + } + assert cache[2] == {'disk_io': {'disk1': {('disk1', 2)}}} + + def test_cache_changing_keys(self): + input = {'disk1': nt(5, 5, 5)} + wrap_numbers(input, 'disk_io') + input = {'disk1': nt(5, 5, 5), 'disk2': nt(7, 7, 7)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + assert cache[1] == { + 'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 0} + } + assert cache[2] == {'disk_io': {}} + + def test_cache_clear(self): + input = {'disk1': nt(5, 5, 5)} + wrap_numbers(input, 'disk_io') + wrap_numbers(input, 'disk_io') + wrap_numbers.cache_clear('disk_io') + assert wrap_numbers.cache_info() == ({}, {}, {}) + wrap_numbers.cache_clear('disk_io') + wrap_numbers.cache_clear('?!?') + + @skipif(not HAS_NET_IO_COUNTERS, reason="not supported") + def test_cache_clear_public_apis(self): + if not psutil.disk_io_counters() or not psutil.net_io_counters(): + return pytest.skip("no disks or NICs available") + psutil.disk_io_counters() + psutil.net_io_counters() + caches = wrap_numbers.cache_info() + for cache in caches: + assert 'psutil.disk_io_counters' in cache + assert 'psutil.net_io_counters' in cache + + psutil.disk_io_counters.cache_clear() + caches = wrap_numbers.cache_info() + for cache in caches: + assert 'psutil.net_io_counters' in cache + assert 'psutil.disk_io_counters' not in cache + + psutil.net_io_counters.cache_clear() + caches = wrap_numbers.cache_info() + assert caches == ({}, {}, {}) + + +# =================================================================== +# --- Test setup.py +# =================================================================== + + +@skipif(not POSIX, reason="POSIX only") +class TestSetupPy(PsutilTestCase): + @staticmethod + def import_setup_py(): + path = os.path.join(ROOT_DIR, "setup.py") + if not os.path.exists(path): + return pytest.skip("setup.py not available") + return import_module_by_path(path) + + def test_num_cpus_env_var(self): + setup = self.import_setup_py() + with mock.patch.dict(os.environ, {"PSUTIL_BUILD_JOBS": "3"}): + assert setup.num_cpus() == 3 + # Never return 0, else ThreadPoolExecutor() raises ValueError. + with mock.patch.dict(os.environ, {"PSUTIL_BUILD_JOBS": "0"}): + assert setup.num_cpus() == 1 + + def test_num_cpus_default(self): + setup = self.import_setup_py() + with mock.patch.dict(os.environ, clear=True): + assert setup.num_cpus() >= 1 + + def test_get_cc(self): + setup = self.import_setup_py() + with mock.patch.dict(os.environ, {"CC": "gcc -pthread"}): + assert setup.get_cc() == ["gcc", "-pthread"] + + @staticmethod + def run_instructions(setup, **flags): + """Call print_install_instructions() and return what it wrote + to stderr. + """ + with contextlib.ExitStack() as stack: + for name, value in flags.items(): + stack.enter_context(mock.patch.object(setup, name, value)) + f = stack.enter_context(contextlib.redirect_stderr(io.StringIO())) + setup.print_install_instructions() + return f.getvalue() + + def test_instructions_are_silent_if_toolchain_is_ok(self): + # Else any unrelated build failure would wrongly blame the + # compiler or the headers. + setup = self.import_setup_py() + out = self.run_instructions( + setup, has_compiler=lambda: True, has_python_h=lambda: True + ) + assert out == "" + + def test_instructions_without_compiler(self): + setup = self.import_setup_py() + out = self.run_instructions( + setup, + has_compiler=lambda: False, + MACOS=False, + AIX=False, + PYPY=False, + ) + assert "C compiler is not installed" in out + assert "install-sysdeps.sh" in out + + def test_instructions_on_macos(self): + setup = self.import_setup_py() + out = self.run_instructions( + setup, has_compiler=lambda: False, MACOS=True + ) + assert "xcode-select --install" in out + assert "install-sysdeps.sh" not in out + + def test_instructions_without_headers(self): + # No command is suggested on platforms install-sysdeps.sh + # doesn't cover. + setup = self.import_setup_py() + out = self.run_instructions( + setup, + has_compiler=lambda: True, + has_python_h=lambda: False, + MACOS=False, + AIX=True, + PYPY=False, + ) + assert "header files are not installed" in out + assert "Try running" not in out + + def test_detection_without_compiler(self): + setup = self.import_setup_py() + with mock.patch.dict(os.environ, {"CC": "psutil-no-such-cc"}): + assert setup.has_compiler() is False + assert setup.has_python_h() is False diff --git a/tests/test_osx.py b/tests/test_osx.py new file mode 100755 index 0000000000..a354d2c311 --- /dev/null +++ b/tests/test_osx.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""macOS specific tests.""" + +import re +import time + +import psutil +from psutil import MACOS +from psutil import _psutil + +from . import AARCH64 +from . import HAS_BATTERY +from . import TOLERANCE_DISK_USAGE +from . import TOLERANCE_SYS_MEM +from . import PsutilTestCase +from . import retry_on_failure +from . import sh +from . import skipif +from . import spawn_subproc +from . import terminate + + +def sysctl(cmdline): + """Expects a sysctl command with an argument and parse the result + returning only the value of interest. + """ + out = sh(cmdline) + result = out.split()[1] + try: + return int(result) + except ValueError: + return result + + +def vm_stat(field): + """Wrapper around 'vm_stat' cmdline utility.""" + out = sh('vm_stat') + for line in out.split('\n'): + if field in line: + break + else: + raise ValueError("line not found") + return int(re.search(r'\d+', line).group(0)) * _psutil.getpagesize() + + +@skipif(not MACOS, reason="MACOS only") +class MacosTestCase(PsutilTestCase): + pass + + +# ===================================================================== +# --- Process APIs (most are tested in test_posix.py) +# ===================================================================== + + +class TestProcess(MacosTestCase): + + @classmethod + def setUpClass(cls): + cls.pid = spawn_subproc().pid + + @classmethod + def tearDownClass(cls): + terminate(cls.pid) + + def test_create_time(self): + output = sh(f"ps -o lstart -p {self.pid}") + start_ps = output.replace('STARTED', '').strip() + hhmmss = start_ps.split(' ')[-2] + year = start_ps.split(' ')[-1] + start_psutil = psutil.Process(self.pid).create_time() + assert hhmmss == time.strftime( + "%H:%M:%S", time.localtime(start_psutil) + ) + assert year == time.strftime("%Y", time.localtime(start_psutil)) + + +# ===================================================================== +# --- Test system APIs +# ===================================================================== + + +class TestVirtualMemory(MacosTestCase): + + def test_total(self): + sysctl_hwphymem = sysctl('sysctl hw.memsize') + assert sysctl_hwphymem == psutil.virtual_memory().total + + @retry_on_failure + def test_free(self): + vmstat_val = vm_stat("free") + psutil_val = psutil.virtual_memory().free + assert abs(psutil_val - vmstat_val) < TOLERANCE_SYS_MEM * 5 + + @retry_on_failure + def test_active(self): + vmstat_val = vm_stat("active") + psutil_val = psutil.virtual_memory().active + assert abs(psutil_val - vmstat_val) < TOLERANCE_SYS_MEM * 5 + + @retry_on_failure + def test_inactive(self): + vmstat_val = vm_stat("inactive") + psutil_val = psutil.virtual_memory().inactive + assert abs(psutil_val - vmstat_val) < TOLERANCE_SYS_MEM * 5 + + @retry_on_failure + def test_wired(self): + vmstat_val = vm_stat("wired") + psutil_val = psutil.virtual_memory().wired + assert abs(psutil_val - vmstat_val) < TOLERANCE_SYS_MEM * 5 + + +class TestSwapMemory(MacosTestCase): + + @staticmethod + def parse_swapusage(out): + # Parse 'sysctl vm.swapusage' output into bytes. + # E.g. 'total = 2.00G' -> 2147483648. + units = {"K": 1024, "M": 1024**2, "G": 1024**3} + ret = {} + for key in ("total", "used", "free"): + m = re.search(rf"{key}\s*=\s*([0-9.]+)([KMG])", out) + ret[key] = int(float(m.group(1)) * units[m.group(2)]) + return ret + + def test_total(self): + out = sh("sysctl vm.swapusage") + sysctl_val = self.parse_swapusage(out)["total"] + # 0.01M display precision = ~10KB rounding + assert abs(psutil.swap_memory().total - sysctl_val) < 100 * 1024 + + @retry_on_failure + def test_used(self): + out = sh("sysctl vm.swapusage") + sysctl_val = self.parse_swapusage(out)["used"] + assert abs(psutil.swap_memory().used - sysctl_val) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_free(self): + out = sh("sysctl vm.swapusage") + sysctl_val = self.parse_swapusage(out)["free"] + assert abs(psutil.swap_memory().free - sysctl_val) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_sin(self): + vmstat_val = vm_stat("Pageins") + psutil_val = psutil.swap_memory().sin + assert abs(psutil_val - vmstat_val) < TOLERANCE_SYS_MEM + + @retry_on_failure + def test_sout(self): + vmstat_val = vm_stat("Pageout") + psutil_val = psutil.swap_memory().sout + assert abs(psutil_val - vmstat_val) < TOLERANCE_SYS_MEM + + +class TestCpuAPIs(MacosTestCase): + + def test_cpu_count_logical(self): + num = sysctl("sysctl hw.logicalcpu") + assert num == psutil.cpu_count(logical=True) + + def test_cpu_count_cores(self): + num = sysctl("sysctl hw.physicalcpu") + assert num == psutil.cpu_count(logical=False) + + # On Apple Silicon cpu_freq() reads IOKit, and there's no sysctl + # (or any other CLI tool) to compare it against. + @skipif(AARCH64, reason="no hw.cpufrequency sysctl on Apple Silicon") + def test_cpu_freq(self): + freq = psutil.cpu_freq() + assert freq.current * 1000 * 1000 == sysctl("sysctl hw.cpufrequency") + assert freq.min * 1000 * 1000 == sysctl("sysctl hw.cpufrequency_min") + assert freq.max * 1000 * 1000 == sysctl("sysctl hw.cpufrequency_max") + + +class TestDiskAPIs(MacosTestCase): + + @retry_on_failure + def test_disk_partitions(self): + # test psutil.disk_usage() and psutil.disk_partitions() + # against "df -a" + def df(path): + out = sh(f'df -k "{path}"').strip() + lines = out.split('\n') + lines.pop(0) + line = lines.pop(0) + dev, total, used, free = line.split()[:4] + if dev == 'none': + dev = '' + total = int(total) * 1024 + used = int(used) * 1024 + free = int(free) * 1024 + return dev, total, used, free + + for part in psutil.disk_partitions(all=False): + usage = psutil.disk_usage(part.mountpoint) + dev, total, used, free = df(part.mountpoint) + assert part.device == dev + assert usage.total == total + assert abs(usage.free - free) < TOLERANCE_DISK_USAGE + assert abs(usage.used - used) < TOLERANCE_DISK_USAGE + + +class TestNetAPIs(MacosTestCase): + + def test_net_if_stats(self): + for name, stats in psutil.net_if_stats().items(): + try: + out = sh(f"ifconfig {name}") + except RuntimeError: + pass + else: + assert stats.isup == ('RUNNING' in out), out + assert stats.mtu == int(re.findall(r'mtu (\d+)', out)[0]) + + @retry_on_failure + def test_net_io_counters(self): + out = sh("netstat -ib") + netstat = {} + for line in out.splitlines(): + fields = line.split() + if len(fields) < 10 or "